Commit 54f7ce68 authored by Matija Obreza's avatar Matija Obreza

Merge branch 'es-more'

* es-more:
  @IgnoreField skips serialization for ES index
  Refactored ES methods
parents f7e2c1f0 0f9291d5
......@@ -14,10 +14,10 @@ The genesys container can be configured via environment variables (`-e` docker f
## Running genesys2-server with maven
- Clone genesys2-server to your computer
- Clone the project code: `git clone ...`
- Create a blank mysql database
- Configure genesys2-server database connection settings
- Start Jetty with `mvn -Dspring.profiles.active=dev jetty:run`
- Configure genesys2-server database connection settings (see below)
- Start Genesys server with `mvn jetty:run`
### Creating a mysql database
......@@ -31,7 +31,7 @@ CREATE DATABASE genesys DEFAULT CHARSET UTF8;
GRANT ALL ON genesys.* TO 'genesys'@'localhost' IDENTIFIED BY 'pwd';
```
Change the settings in `src/main/resources/spring/spring.properties`:
Change the settings in a new file `src/main/resources/genesys.properties`:
```.properties
# genesys2 with mysql database
......
......@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
......@@ -83,8 +84,8 @@ public class SearchController {
QueryBuilder extraFilters = filtersFromDataset(filters);
Map<Class<BasicModel>, List<BasicModel>> hitsByEntity = elasticsearch.search(extraFilters, newHashSet(Crop.class, Partner.class,
AccessionIdentifier.class, Descriptor.class), searchQuery);
Set<Class<? extends BasicModel>> clazzes = newHashSet(Crop.class, Partner.class, Descriptor.class);
Map<Class<? extends BasicModel>, List<? extends BasicModel>> hitsByEntity = elasticsearch.search(extraFilters, searchQuery, clazzes);
Map<String, SearchResults<?>> suggestions = new HashMap<>();
suggestions.put("search.group.crop", SearchResults.from("code", Arrays.asList("crop"), hitsByEntity.get(Crop.class)));
......@@ -93,7 +94,7 @@ public class SearchController {
suggestions.put("search.group.descriptor", SearchResults.from("uuid", Arrays.asList("descriptor.uuid"), hitsByEntity.get(Descriptor.class)));
// Search datasets
suggestions.put("search.matches", SearchResults.from("uuid", Arrays.asList("uuid"), elasticsearch.search(extraFilters, Dataset.class, searchQuery)));
suggestions.put("search.matches", SearchResults.from("uuid", Arrays.asList("uuid"), elasticsearch.search(extraFilters, searchQuery, Dataset.class)));
return suggestions;
}
......
......@@ -13,16 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.custom.elasticsearch;
package org.genesys.custom.elasticsearch;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.elasticsearch.ElasticsearchException;
import java.io.IOException;
import static org.genesys.custom.elasticsearch.CustomMappingBuilder.*;
import static org.genesys.catalog.custom.elasticsearch.CustomMappingBuilder.buildMapping;
import java.io.IOException;
/**
......
......@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.custom.elasticsearch;
package org.genesys.custom.elasticsearch;
import static org.apache.commons.lang.StringUtils.*;
import static org.elasticsearch.common.xcontent.XContentFactory.*;
......@@ -57,8 +57,8 @@ import org.springframework.data.util.TypeInformation;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
/**
* Modified MappingBuilder that includes all simple (non-entity) types and forces index
* to not use dynamic mappings.
* Modified MappingBuilder that includes all simple (non-entity) types and
* forces index to not use dynamic mappings.
*
* @author Rizwan Idrees
* @author Mohsin Husen
......@@ -71,64 +71,64 @@ import com.fasterxml.jackson.annotation.JsonUnwrapped;
* @author Matija Obreza
*/
class CustomMappingBuilder {
/** The Constant LOG. */
public static final Logger LOG = LoggerFactory.getLogger(CustomMappingBuilder.class);
/** The Constant FIELD_STORE. */
public static final String FIELD_STORE = "store";
/** The Constant FIELD_TYPE. */
public static final String FIELD_TYPE = "type";
/** The Constant FIELD_INDEX. */
public static final String FIELD_INDEX = "index";
/** The Constant FIELD_FORMAT. */
public static final String FIELD_FORMAT = "format";
/** The Constant FIELD_SEARCH_ANALYZER. */
public static final String FIELD_SEARCH_ANALYZER = "search_analyzer";
/** The Constant FIELD_INDEX_ANALYZER. */
public static final String FIELD_INDEX_ANALYZER = "analyzer";
/** The Constant FIELD_DYNAMIC. */
public static final String FIELD_DYNAMIC = "dynamic";
/** The Constant FIELD_PROPERTIES. */
public static final String FIELD_PROPERTIES = "properties";
/** The Constant FIELD_PARENT. */
public static final String FIELD_PARENT = "_parent";
/** The Constant COMPLETION_PAYLOADS. */
public static final String COMPLETION_PAYLOADS = "payloads";
/** The Constant COMPLETION_PRESERVE_SEPARATORS. */
public static final String COMPLETION_PRESERVE_SEPARATORS = "preserve_separators";
/** The Constant COMPLETION_PRESERVE_POSITION_INCREMENTS. */
public static final String COMPLETION_PRESERVE_POSITION_INCREMENTS = "preserve_position_increments";
/** The Constant COMPLETION_MAX_INPUT_LENGTH. */
public static final String COMPLETION_MAX_INPUT_LENGTH = "max_input_length";
/** The Constant INDEX_VALUE_NOT_ANALYZED. */
public static final String INDEX_VALUE_NOT_ANALYZED = "not_analyzed";
/** The Constant TYPE_VALUE_STRING. */
public static final String TYPE_VALUE_STRING = "string";
/** The Constant TYPE_VALUE_GEO_POINT. */
public static final String TYPE_VALUE_GEO_POINT = "geo_point";
/** The Constant TYPE_VALUE_COMPLETION. */
public static final String TYPE_VALUE_COMPLETION = "completion";
/** The Constant TYPE_VALUE_GEO_HASH_PREFIX. */
public static final String TYPE_VALUE_GEO_HASH_PREFIX = "geohash_prefix";
/** The Constant TYPE_VALUE_GEO_HASH_PRECISION. */
public static final String TYPE_VALUE_GEO_HASH_PRECISION = "geohash_precision";
......@@ -151,7 +151,7 @@ class CustomMappingBuilder {
if (hasText(parentType)) {
mapping.startObject(FIELD_PARENT).field(FIELD_TYPE, parentType).endObject();
}
// Dynamic
mapping.field(FIELD_DYNAMIC, false);
......@@ -164,7 +164,7 @@ class CustomMappingBuilder {
}
private static void mapEntity(Set<Class<?>> circularReferences, XContentBuilder xContentBuilder, Class clazz, boolean isRootObject, String idFieldName,
String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType, Field fieldAnnotation) throws IOException {
String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType, Field fieldAnnotation) throws IOException {
if (circularReferences.contains(clazz)) {
LOG.info("Circular reference detected class={} in {}", clazz, circularReferences);
......@@ -172,7 +172,7 @@ class CustomMappingBuilder {
} else {
circularReferences.add(clazz);
}
java.lang.reflect.Field[] fields = retrieveFields(clazz);
if (!isRootObject && (isAnyPropertyAnnotatedAsField(fields) || nestedOrObjectField)) {
......@@ -192,7 +192,7 @@ class CustomMappingBuilder {
if (field.isAnnotationPresent(Transient.class) || isInIgnoreFields(field, fieldAnnotation)) {
// Keep transient or ignored fields that have Field annotation
if (! field.isAnnotationPresent(Field.class))
if (!field.isAnnotationPresent(Field.class))
continue;
}
......@@ -222,7 +222,8 @@ class CustomMappingBuilder {
JsonUnwrapped isUnwrapped = field.getAnnotation(JsonUnwrapped.class);
includedInRoot = isUnwrapped != null ? true : false;
}
mapEntity(circularReferences, xContentBuilder, getFieldType(field), includedInRoot, EMPTY, field.getName(), nestedOrObject, singleField.type(), field.getAnnotation(Field.class));
mapEntity(circularReferences, xContentBuilder, getFieldType(field), includedInRoot, EMPTY, field.getName(), nestedOrObject, singleField.type(), field.getAnnotation(
Field.class));
if (nestedOrObject) {
continue;
}
......@@ -252,7 +253,7 @@ class CustomMappingBuilder {
// Add _class field to entities
xContentBuilder.startObject("_class").field(FIELD_TYPE, TYPE_VALUE_STRING).field(FIELD_STORE, true).endObject();
if (!isRootObject && isAnyPropertyAnnotatedAsField(fields) || nestedOrObjectField) {
xContentBuilder.endObject().endObject();
}
......@@ -274,17 +275,14 @@ class CustomMappingBuilder {
do {
fields.addAll(Arrays.asList(targetClass.getDeclaredFields()));
targetClass = targetClass.getSuperclass();
}
while (targetClass != null && targetClass != Object.class);
} while (targetClass != null && targetClass != Object.class);
return fields.toArray(new java.lang.reflect.Field[fields.size()]);
}
private static boolean isAnnotated(java.lang.reflect.Field field) {
return field.getAnnotation(Field.class) != null ||
field.getAnnotation(MultiField.class) != null ||
field.getAnnotation(GeoPointField.class) != null ||
field.getAnnotation(CompletionField.class) != null;
return field.getAnnotation(Field.class) != null || field.getAnnotation(MultiField.class) != null || field.getAnnotation(GeoPointField.class) != null || field.getAnnotation(
CompletionField.class) != null;
}
private static void applyGeoPointFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field) throws IOException {
......@@ -326,11 +324,8 @@ class CustomMappingBuilder {
xContentBuilder.endObject();
}
private static void applyDefaultIdFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field)
throws IOException {
xContentBuilder.startObject(field.getName())
.field(FIELD_TYPE, TYPE_VALUE_STRING)
.field(FIELD_INDEX, INDEX_VALUE_NOT_ANALYZED);
private static void applyDefaultIdFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field) throws IOException {
xContentBuilder.startObject(field.getName()).field(FIELD_TYPE, TYPE_VALUE_STRING).field(FIELD_INDEX, INDEX_VALUE_NOT_ANALYZED);
xContentBuilder.endObject();
}
......@@ -339,19 +334,18 @@ class CustomMappingBuilder {
*
* @throws IOException
*/
private static void addSingleFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field,
Field fieldAnnotation, boolean nestedOrObjectField) throws IOException {
private static void addSingleFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field, Field fieldAnnotation, boolean nestedOrObjectField)
throws IOException {
LOG.trace("addSingleFieldMapping {}#{}", field.getDeclaringClass(), field.getName());
xContentBuilder.startObject(field.getName());
if(!nestedOrObjectField) {
if (!nestedOrObjectField) {
xContentBuilder.field(FIELD_STORE, fieldAnnotation == null ? true : fieldAnnotation.store());
}
if (fieldAnnotation != null) {
if (FieldType.Auto != fieldAnnotation.type()) {
xContentBuilder.field(FIELD_TYPE, fieldAnnotation.type().name().toLowerCase());
if (FieldType.Date == fieldAnnotation.type() && DateFormat.none != fieldAnnotation.format()) {
xContentBuilder.field(FIELD_FORMAT, DateFormat.custom == fieldAnnotation.format()
? fieldAnnotation.pattern() : fieldAnnotation.format());
xContentBuilder.field(FIELD_FORMAT, DateFormat.custom == fieldAnnotation.format() ? fieldAnnotation.pattern() : fieldAnnotation.format());
}
} else {
xContentBuilder.field(FIELD_TYPE, typeForField(field).name().toLowerCase());
......@@ -379,13 +373,14 @@ class CustomMappingBuilder {
private static FieldType typeForField(java.lang.reflect.Field field) {
if (isCollection(field)) {
ParameterizedType paramType = (ParameterizedType) field.getGenericType();
Class<?> paramClass = (Class<?>) paramType.getActualTypeArguments()[0];
return typeForClass(paramClass, field);
ParameterizedType paramType = (ParameterizedType) field.getGenericType();
Class<?> paramClass = (Class<?>) paramType.getActualTypeArguments()[0];
return typeForClass(paramClass, field);
} else {
return typeForClass(field.getType(), field);
}
}
private static FieldType typeForClass(Class<?> clazz, java.lang.reflect.Field field) {
if (String.class.equals(clazz)) {
return FieldType.String;
......@@ -410,7 +405,7 @@ class CustomMappingBuilder {
return FieldType.String;
}
}
private static FieldIndex indexForField(java.lang.reflect.Field field) {
if (isCollection(field)) {
ParameterizedType paramType = (ParameterizedType) field.getGenericType();
......@@ -430,26 +425,26 @@ class CustomMappingBuilder {
analyzed = true;
} else if (jpaColumn != null) {
if (jpaColumn.length() > 100) {
analyzed=true;
analyzed = true;
}
}
return analyzed ? FieldIndex.analyzed : FieldIndex.not_analyzed;
}
// else if (Boolean.TYPE.equals(clazz) || Boolean.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Long.TYPE.equals(clazz) || Long.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Double.TYPE.equals(clazz) || Double.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Float.TYPE.equals(clazz) || Float.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Integer.TYPE.equals(clazz) || Integer.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Date.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Number.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// }
// else if (Boolean.TYPE.equals(clazz) || Boolean.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Long.TYPE.equals(clazz) || Long.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Double.TYPE.equals(clazz) || Double.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Float.TYPE.equals(clazz) || Float.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Integer.TYPE.equals(clazz) || Integer.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Date.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Number.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// }
else if (clazz.isEnum()) {
return FieldIndex.not_analyzed;
} else {
......@@ -458,16 +453,14 @@ class CustomMappingBuilder {
}
}
/**
* Apply mapping for a single nested @Field annotation
*
* @throws IOException
*/
private static void addNestedFieldMapping(XContentBuilder builder, java.lang.reflect.Field field,
InnerField annotation) throws IOException {
private static void addNestedFieldMapping(XContentBuilder builder, java.lang.reflect.Field field, InnerField annotation) throws IOException {
builder.startObject(annotation.suffix());
//builder.field(FIELD_STORE, annotation.store());
// builder.field(FIELD_STORE, annotation.store());
if (FieldType.Auto != annotation.type()) {
builder.field(FIELD_TYPE, annotation.type().name().toLowerCase());
}
......@@ -488,13 +481,12 @@ class CustomMappingBuilder {
*
* @throws IOException
*/
private static void addMultiFieldMapping(XContentBuilder builder, java.lang.reflect.Field field,
MultiField annotation, boolean nestedOrObjectField) throws IOException {
private static void addMultiFieldMapping(XContentBuilder builder, java.lang.reflect.Field field, MultiField annotation, boolean nestedOrObjectField) throws IOException {
builder.startObject(field.getName());
builder.field(FIELD_TYPE, "multi_field");
builder.startObject("fields");
//add standard field
addSingleFieldMapping(builder, field, annotation.mainField(),nestedOrObjectField);
// add standard field
addSingleFieldMapping(builder, field, annotation.mainField(), nestedOrObjectField);
for (InnerField innerField : annotation.otherFields()) {
addNestedFieldMapping(builder, field, innerField);
}
......@@ -509,7 +501,7 @@ class CustomMappingBuilder {
* @return true, if is entity
*/
protected static boolean isEntity(java.lang.reflect.Field field) {
TypeInformation<?> typeInformation = ClassTypeInformation.from(field.getType());
Class<?> clazz = getFieldType(field);
boolean isComplexType = !SIMPLE_TYPE_HOLDER.isSimpleType(clazz);
......@@ -523,7 +515,7 @@ class CustomMappingBuilder {
* @return true, if is map
*/
protected static boolean isMap(java.lang.reflect.Field field) {
TypeInformation<?> typeInformation = ClassTypeInformation.from(field.getType());
return typeInformation.isMap();
}
......@@ -535,7 +527,7 @@ class CustomMappingBuilder {
* @return true, if is collection
*/
protected static boolean isCollection(java.lang.reflect.Field field) {
TypeInformation<?> typeInformation = ClassTypeInformation.from(field.getType());
return typeInformation.isCollectionLike();
}
......@@ -547,11 +539,11 @@ class CustomMappingBuilder {
* @return the field type
*/
protected static Class<?> getFieldType(java.lang.reflect.Field field) {
return ClassTypeInformation.from(field.getDeclaringClass()) //
.getProperty(field.getName()) //
.getActualType() //
.getType();
.getProperty(field.getName()) //
.getActualType() //
.getType();
}
private static boolean isAnyPropertyAnnotatedAsField(java.lang.reflect.Field[] fields) {
......@@ -573,6 +565,11 @@ class CustomMappingBuilder {
if (field.getName().equals("serialVersionUID")) {
return true;
}
if (field.isAnnotationPresent(IgnoreField.class)) {
// System.err.println("Not indexing @IgnoreField on " +
// field.getDeclaringClass() + "#" + field.getName());
return true;
}
if (null != parentFieldAnnotation) {
String[] ignoreFields = parentFieldAnnotation.ignoreFields();
return Arrays.asList(ignoreFields).contains(field.getName());
......
......@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.custom.elasticsearch;
package org.genesys.custom.elasticsearch;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.node.NodeClient;
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.custom.elasticsearch;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Allows for excluding fields from being mapped and sent to ES.
*
* @author Matija Obreza
* @since 1 Aug 2018
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.ANNOTATION_TYPE, ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.FIELD})
@Documented
@Inherited
public @interface IgnoreField {
}
......@@ -336,7 +336,7 @@ public class AccessionController extends ApiBaseController {
@RequestMapping(value = "/search", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody List<Accession> search(@RequestParam("page") final int page, @RequestParam("query") final String query) throws SearchException {
return elasticService.search(null, Accession.class, StringUtils.defaultIfBlank(query, "*")); //, new PageRequest(page - 1, 20));
return elasticService.search(null, StringUtils.defaultIfBlank(query, "*"), Accession.class); //, new PageRequest(page - 1, 20));
}
// FIXME Not using institute...
......@@ -349,7 +349,7 @@ public class AccessionController extends ApiBaseController {
}
query = StringUtils.defaultIfBlank(query, "*");
// TODO filer by inst
return elasticService.search(null, Accession.class, query); // , new PageRequest(page - 1, 20));
return elasticService.search(null, query, Accession.class); // , new PageRequest(page - 1, 20));
}
@RequestMapping(value = "/{id}", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
......
......@@ -16,6 +16,8 @@
package org.genesys2.server.component.elastic;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
......@@ -24,18 +26,23 @@ import java.util.concurrent.BlockingQueue;
import javax.annotation.Resource;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.After;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.genesys.blocks.model.BasicModel;
import org.genesys2.server.service.ElasticsearchService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.aop.framework.Advised;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.stereotype.Component;
/**
* AspectJ powered listener on repository save and delete operations
* adds indexed entities to re-index queue, handled by {@link ElasticReindexProcessor}.
* AspectJ powered listener on repository save and delete operations adds
* indexed entities to re-index queue, handled by
* {@link ElasticReindexProcessor}.
*/
@Aspect
@Component
......@@ -44,10 +51,13 @@ public class ElasticJPAListener {
private Set<Object> includedClasses;
private Set<Object> ignoredClasses;
@Resource
private BlockingQueue<ElasticReindex> elasticReindexQueue;
@Autowired
private ElasticsearchService elasticsearchService;
/**
* Instantiates a new elastic JPA listener.
*/
......@@ -91,6 +101,35 @@ public class ElasticJPAListener {
}
}
/**
* Delete all
*
* @param joinPoint the join point
*/
@After(value = "execution(* org.springframework.data.jpa.repository.JpaRepository.deleteAll())")
public void afterDeleteAll(final JoinPoint joinPoint) {
try {
LOG.trace("JPA afterDeleteAll: {} {}", joinPoint.toLongString(), joinPoint.getTarget());
Object proxy = joinPoint.getTarget();
if (proxy instanceof Advised) {
Advised x = (Advised) proxy;
for (Class<?> foo : x.getProxiedInterfaces()) {
for (Type generic : foo.getGenericInterfaces()) {
if (generic instanceof ParameterizedType) {
Class<?> paramClass = (Class<?>) ((ParameterizedType) generic).getActualTypeArguments()[0];
if (BasicModel.class.isAssignableFrom(paramClass)) {