Commit 0f9291d5 authored by Matija Obreza's avatar Matija Obreza

@IgnoreField skips serialization for ES index

- Uses custom introspector and mapping generator
parent 52c6d72b
......@@ -14,10 +14,10 @@ The genesys container can be configured via environment variables (`-e` docker f
## Running genesys2-server with maven
- Clone genesys2-server to your computer
- Clone the project code: `git clone ...`
- Create a blank mysql database
- Configure genesys2-server database connection settings
- Start Jetty with `mvn -Dspring.profiles.active=dev jetty:run`
- Configure genesys2-server database connection settings (see below)
- Start Genesys server with `mvn jetty:run`
### Creating a mysql database
......@@ -31,7 +31,7 @@ CREATE DATABASE genesys DEFAULT CHARSET UTF8;
GRANT ALL ON genesys.* TO 'genesys'@'localhost' IDENTIFIED BY 'pwd';
```
Change the settings in `src/main/resources/spring/spring.properties`:
Change the settings in a new file `src/main/resources/genesys.properties`:
```.properties
# genesys2 with mysql database
......
......@@ -13,16 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.custom.elasticsearch;
package org.genesys.custom.elasticsearch;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.elasticsearch.ElasticsearchException;
import java.io.IOException;
import static org.genesys.custom.elasticsearch.CustomMappingBuilder.*;
import static org.genesys.catalog.custom.elasticsearch.CustomMappingBuilder.buildMapping;
import java.io.IOException;
/**
......
......@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.custom.elasticsearch;
package org.genesys.custom.elasticsearch;
import static org.apache.commons.lang.StringUtils.*;
import static org.elasticsearch.common.xcontent.XContentFactory.*;
......@@ -57,8 +57,8 @@ import org.springframework.data.util.TypeInformation;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
/**
* Modified MappingBuilder that includes all simple (non-entity) types and forces index
* to not use dynamic mappings.
* Modified MappingBuilder that includes all simple (non-entity) types and
* forces index to not use dynamic mappings.
*
* @author Rizwan Idrees
* @author Mohsin Husen
......@@ -71,64 +71,64 @@ import com.fasterxml.jackson.annotation.JsonUnwrapped;
* @author Matija Obreza
*/
class CustomMappingBuilder {
/** The Constant LOG. */
public static final Logger LOG = LoggerFactory.getLogger(CustomMappingBuilder.class);
/** The Constant FIELD_STORE. */
public static final String FIELD_STORE = "store";
/** The Constant FIELD_TYPE. */
public static final String FIELD_TYPE = "type";
/** The Constant FIELD_INDEX. */
public static final String FIELD_INDEX = "index";
/** The Constant FIELD_FORMAT. */
public static final String FIELD_FORMAT = "format";
/** The Constant FIELD_SEARCH_ANALYZER. */
public static final String FIELD_SEARCH_ANALYZER = "search_analyzer";
/** The Constant FIELD_INDEX_ANALYZER. */
public static final String FIELD_INDEX_ANALYZER = "analyzer";
/** The Constant FIELD_DYNAMIC. */
public static final String FIELD_DYNAMIC = "dynamic";
/** The Constant FIELD_PROPERTIES. */
public static final String FIELD_PROPERTIES = "properties";
/** The Constant FIELD_PARENT. */
public static final String FIELD_PARENT = "_parent";
/** The Constant COMPLETION_PAYLOADS. */
public static final String COMPLETION_PAYLOADS = "payloads";
/** The Constant COMPLETION_PRESERVE_SEPARATORS. */
public static final String COMPLETION_PRESERVE_SEPARATORS = "preserve_separators";
/** The Constant COMPLETION_PRESERVE_POSITION_INCREMENTS. */
public static final String COMPLETION_PRESERVE_POSITION_INCREMENTS = "preserve_position_increments";
/** The Constant COMPLETION_MAX_INPUT_LENGTH. */
public static final String COMPLETION_MAX_INPUT_LENGTH = "max_input_length";
/** The Constant INDEX_VALUE_NOT_ANALYZED. */
public static final String INDEX_VALUE_NOT_ANALYZED = "not_analyzed";
/** The Constant TYPE_VALUE_STRING. */
public static final String TYPE_VALUE_STRING = "string";
/** The Constant TYPE_VALUE_GEO_POINT. */
public static final String TYPE_VALUE_GEO_POINT = "geo_point";
/** The Constant TYPE_VALUE_COMPLETION. */
public static final String TYPE_VALUE_COMPLETION = "completion";
/** The Constant TYPE_VALUE_GEO_HASH_PREFIX. */
public static final String TYPE_VALUE_GEO_HASH_PREFIX = "geohash_prefix";
/** The Constant TYPE_VALUE_GEO_HASH_PRECISION. */
public static final String TYPE_VALUE_GEO_HASH_PRECISION = "geohash_precision";
......@@ -151,7 +151,7 @@ class CustomMappingBuilder {
if (hasText(parentType)) {
mapping.startObject(FIELD_PARENT).field(FIELD_TYPE, parentType).endObject();
}
// Dynamic
mapping.field(FIELD_DYNAMIC, false);
......@@ -164,7 +164,7 @@ class CustomMappingBuilder {
}
private static void mapEntity(Set<Class<?>> circularReferences, XContentBuilder xContentBuilder, Class clazz, boolean isRootObject, String idFieldName,
String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType, Field fieldAnnotation) throws IOException {
String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType, Field fieldAnnotation) throws IOException {
if (circularReferences.contains(clazz)) {
LOG.info("Circular reference detected class={} in {}", clazz, circularReferences);
......@@ -172,7 +172,7 @@ class CustomMappingBuilder {
} else {
circularReferences.add(clazz);
}
java.lang.reflect.Field[] fields = retrieveFields(clazz);
if (!isRootObject && (isAnyPropertyAnnotatedAsField(fields) || nestedOrObjectField)) {
......@@ -192,7 +192,7 @@ class CustomMappingBuilder {
if (field.isAnnotationPresent(Transient.class) || isInIgnoreFields(field, fieldAnnotation)) {
// Keep transient or ignored fields that have Field annotation
if (! field.isAnnotationPresent(Field.class))
if (!field.isAnnotationPresent(Field.class))
continue;
}
......@@ -222,7 +222,8 @@ class CustomMappingBuilder {
JsonUnwrapped isUnwrapped = field.getAnnotation(JsonUnwrapped.class);
includedInRoot = isUnwrapped != null ? true : false;
}
mapEntity(circularReferences, xContentBuilder, getFieldType(field), includedInRoot, EMPTY, field.getName(), nestedOrObject, singleField.type(), field.getAnnotation(Field.class));
mapEntity(circularReferences, xContentBuilder, getFieldType(field), includedInRoot, EMPTY, field.getName(), nestedOrObject, singleField.type(), field.getAnnotation(
Field.class));
if (nestedOrObject) {
continue;
}
......@@ -252,7 +253,7 @@ class CustomMappingBuilder {
// Add _class field to entities
xContentBuilder.startObject("_class").field(FIELD_TYPE, TYPE_VALUE_STRING).field(FIELD_STORE, true).endObject();
if (!isRootObject && isAnyPropertyAnnotatedAsField(fields) || nestedOrObjectField) {
xContentBuilder.endObject().endObject();
}
......@@ -274,17 +275,14 @@ class CustomMappingBuilder {
do {
fields.addAll(Arrays.asList(targetClass.getDeclaredFields()));
targetClass = targetClass.getSuperclass();
}
while (targetClass != null && targetClass != Object.class);
} while (targetClass != null && targetClass != Object.class);
return fields.toArray(new java.lang.reflect.Field[fields.size()]);
}
private static boolean isAnnotated(java.lang.reflect.Field field) {
return field.getAnnotation(Field.class) != null ||
field.getAnnotation(MultiField.class) != null ||
field.getAnnotation(GeoPointField.class) != null ||
field.getAnnotation(CompletionField.class) != null;
return field.getAnnotation(Field.class) != null || field.getAnnotation(MultiField.class) != null || field.getAnnotation(GeoPointField.class) != null || field.getAnnotation(
CompletionField.class) != null;
}
private static void applyGeoPointFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field) throws IOException {
......@@ -326,11 +324,8 @@ class CustomMappingBuilder {
xContentBuilder.endObject();
}
private static void applyDefaultIdFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field)
throws IOException {
xContentBuilder.startObject(field.getName())
.field(FIELD_TYPE, TYPE_VALUE_STRING)
.field(FIELD_INDEX, INDEX_VALUE_NOT_ANALYZED);
private static void applyDefaultIdFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field) throws IOException {
xContentBuilder.startObject(field.getName()).field(FIELD_TYPE, TYPE_VALUE_STRING).field(FIELD_INDEX, INDEX_VALUE_NOT_ANALYZED);
xContentBuilder.endObject();
}
......@@ -339,19 +334,18 @@ class CustomMappingBuilder {
*
* @throws IOException
*/
private static void addSingleFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field,
Field fieldAnnotation, boolean nestedOrObjectField) throws IOException {
private static void addSingleFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field, Field fieldAnnotation, boolean nestedOrObjectField)
throws IOException {
LOG.trace("addSingleFieldMapping {}#{}", field.getDeclaringClass(), field.getName());
xContentBuilder.startObject(field.getName());
if(!nestedOrObjectField) {
if (!nestedOrObjectField) {
xContentBuilder.field(FIELD_STORE, fieldAnnotation == null ? true : fieldAnnotation.store());
}
if (fieldAnnotation != null) {
if (FieldType.Auto != fieldAnnotation.type()) {
xContentBuilder.field(FIELD_TYPE, fieldAnnotation.type().name().toLowerCase());
if (FieldType.Date == fieldAnnotation.type() && DateFormat.none != fieldAnnotation.format()) {
xContentBuilder.field(FIELD_FORMAT, DateFormat.custom == fieldAnnotation.format()
? fieldAnnotation.pattern() : fieldAnnotation.format());
xContentBuilder.field(FIELD_FORMAT, DateFormat.custom == fieldAnnotation.format() ? fieldAnnotation.pattern() : fieldAnnotation.format());
}
} else {
xContentBuilder.field(FIELD_TYPE, typeForField(field).name().toLowerCase());
......@@ -379,13 +373,14 @@ class CustomMappingBuilder {
private static FieldType typeForField(java.lang.reflect.Field field) {
if (isCollection(field)) {
ParameterizedType paramType = (ParameterizedType) field.getGenericType();
Class<?> paramClass = (Class<?>) paramType.getActualTypeArguments()[0];
return typeForClass(paramClass, field);
ParameterizedType paramType = (ParameterizedType) field.getGenericType();
Class<?> paramClass = (Class<?>) paramType.getActualTypeArguments()[0];
return typeForClass(paramClass, field);
} else {
return typeForClass(field.getType(), field);
}
}
private static FieldType typeForClass(Class<?> clazz, java.lang.reflect.Field field) {
if (String.class.equals(clazz)) {
return FieldType.String;
......@@ -410,7 +405,7 @@ class CustomMappingBuilder {
return FieldType.String;
}
}
private static FieldIndex indexForField(java.lang.reflect.Field field) {
if (isCollection(field)) {
ParameterizedType paramType = (ParameterizedType) field.getGenericType();
......@@ -430,26 +425,26 @@ class CustomMappingBuilder {
analyzed = true;
} else if (jpaColumn != null) {
if (jpaColumn.length() > 100) {
analyzed=true;
analyzed = true;
}
}
return analyzed ? FieldIndex.analyzed : FieldIndex.not_analyzed;
}
// else if (Boolean.TYPE.equals(clazz) || Boolean.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Long.TYPE.equals(clazz) || Long.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Double.TYPE.equals(clazz) || Double.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Float.TYPE.equals(clazz) || Float.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Integer.TYPE.equals(clazz) || Integer.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Date.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Number.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// }
// else if (Boolean.TYPE.equals(clazz) || Boolean.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Long.TYPE.equals(clazz) || Long.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Double.TYPE.equals(clazz) || Double.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Float.TYPE.equals(clazz) || Float.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Integer.TYPE.equals(clazz) || Integer.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Date.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Number.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// }
else if (clazz.isEnum()) {
return FieldIndex.not_analyzed;
} else {
......@@ -458,16 +453,14 @@ class CustomMappingBuilder {
}
}
/**
* Apply mapping for a single nested @Field annotation
*
* @throws IOException
*/
private static void addNestedFieldMapping(XContentBuilder builder, java.lang.reflect.Field field,
InnerField annotation) throws IOException {
private static void addNestedFieldMapping(XContentBuilder builder, java.lang.reflect.Field field, InnerField annotation) throws IOException {
builder.startObject(annotation.suffix());
//builder.field(FIELD_STORE, annotation.store());
// builder.field(FIELD_STORE, annotation.store());
if (FieldType.Auto != annotation.type()) {
builder.field(FIELD_TYPE, annotation.type().name().toLowerCase());
}
......@@ -488,13 +481,12 @@ class CustomMappingBuilder {
*
* @throws IOException
*/
private static void addMultiFieldMapping(XContentBuilder builder, java.lang.reflect.Field field,
MultiField annotation, boolean nestedOrObjectField) throws IOException {
private static void addMultiFieldMapping(XContentBuilder builder, java.lang.reflect.Field field, MultiField annotation, boolean nestedOrObjectField) throws IOException {
builder.startObject(field.getName());
builder.field(FIELD_TYPE, "multi_field");
builder.startObject("fields");
//add standard field
addSingleFieldMapping(builder, field, annotation.mainField(),nestedOrObjectField);
// add standard field
addSingleFieldMapping(builder, field, annotation.mainField(), nestedOrObjectField);
for (InnerField innerField : annotation.otherFields()) {
addNestedFieldMapping(builder, field, innerField);
}
......@@ -509,7 +501,7 @@ class CustomMappingBuilder {
* @return true, if is entity
*/
protected static boolean isEntity(java.lang.reflect.Field field) {
TypeInformation<?> typeInformation = ClassTypeInformation.from(field.getType());
Class<?> clazz = getFieldType(field);
boolean isComplexType = !SIMPLE_TYPE_HOLDER.isSimpleType(clazz);
......@@ -523,7 +515,7 @@ class CustomMappingBuilder {
* @return true, if is map
*/
protected static boolean isMap(java.lang.reflect.Field field) {
TypeInformation<?> typeInformation = ClassTypeInformation.from(field.getType());
return typeInformation.isMap();
}
......@@ -535,7 +527,7 @@ class CustomMappingBuilder {
* @return true, if is collection
*/
protected static boolean isCollection(java.lang.reflect.Field field) {
TypeInformation<?> typeInformation = ClassTypeInformation.from(field.getType());
return typeInformation.isCollectionLike();
}
......@@ -547,11 +539,11 @@ class CustomMappingBuilder {
* @return the field type
*/
protected static Class<?> getFieldType(java.lang.reflect.Field field) {
return ClassTypeInformation.from(field.getDeclaringClass()) //
.getProperty(field.getName()) //
.getActualType() //
.getType();
.getProperty(field.getName()) //
.getActualType() //
.getType();
}
private static boolean isAnyPropertyAnnotatedAsField(java.lang.reflect.Field[] fields) {
......@@ -573,6 +565,11 @@ class CustomMappingBuilder {
if (field.getName().equals("serialVersionUID")) {
return true;
}
if (field.isAnnotationPresent(IgnoreField.class)) {
// System.err.println("Not indexing @IgnoreField on " +
// field.getDeclaringClass() + "#" + field.getName());
return true;
}
if (null != parentFieldAnnotation) {
String[] ignoreFields = parentFieldAnnotation.ignoreFields();
return Arrays.asList(ignoreFields).contains(field.getName());
......
......@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.custom.elasticsearch;
package org.genesys.custom.elasticsearch;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.node.NodeClient;
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.custom.elasticsearch;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Allows for excluding fields from being mapped and sent to ES.
*
* @author Matija Obreza
* @since 1 Aug 2018
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.ANNOTATION_TYPE, ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.FIELD})
@Documented
@Inherited
public @interface IgnoreField {
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.component.elastic;
import org.genesys.custom.elasticsearch.IgnoreField;
import com.fasterxml.jackson.databind.introspect.Annotated;
import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
/**
* Extra controls to what gets serialized to JSON before sending data over to
* ES.
*
* - Ignores @Field(index = no) properties
*
* @author Matija Obreza
* @since 1 Aug 2018
*/
public class ElasticJacksonAnnotationIntrospector extends JacksonAnnotationIntrospector {
private static final long serialVersionUID = 6225410633030301962L;
/**
* Ignore properties where @Fileld(index = no)
*/
@Override
protected boolean _isIgnorable(Annotated a) {
if (isExcludedFromEs(a)) {
return true;
}
return super._isIgnorable(a);
}
@Override
public Boolean hasRequiredMarker(AnnotatedMember m) {
if (isExcludedFromEs(m)) {
return false;
}
return super.hasRequiredMarker(m);
}
private boolean isExcludedFromEs(Annotated a) {
IgnoreField ignoreField = _findAnnotation(a, IgnoreField.class);
if (ignoreField != null) {
// System.err.println("Should not serialize @IgnoreField on " + a);
}
return ignoreField != null;
}
@Override
public Class<?>[] findViews(Annotated a) {
if (isExcludedFromEs(a)) {
return new Class<?>[] { NotIndexed.class };
}
return super.findViews(a);
}
public static class NotIndexed {
}
}
......@@ -49,6 +49,7 @@ import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
import com.fasterxml.jackson.annotation.JsonView;
......@@ -78,6 +79,7 @@ public abstract class AccessionData extends AuditedVersionedModel implements IdU
@JoinColumn(name = "instituteId")
@JsonView({ JsonViews.Minimal.class })
@Field(type = FieldType.Object)
@JsonIgnoreProperties({"settings"})
private FaoInstitute institute;
@Column(name = "acceNumb", nullable = false, length = 128)
......
......@@ -43,6 +43,7 @@ import org.genesys.blocks.auditlog.annotations.Audited;
import org.genesys.blocks.model.AuditedVersionedModel;
import org.genesys.blocks.model.IdUUID;
import org.genesys.blocks.model.JsonViews;
import org.genesys.custom.elasticsearch.IgnoreField;
import org.hibernate.annotations.Type;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
......@@ -83,7 +84,7 @@ public class AccessionId extends AuditedVersionedModel implements IdUUID {
@JoinColumn(name = "pdciId", unique = true)
@JsonIgnoreProperties({ "accession" })
@JsonView({ JsonViews.Internal.class })
@Field(type = FieldType.Object)
@IgnoreField
private PDCI pdci;
@Column(name = "storage", nullable = false)
......@@ -187,6 +188,7 @@ public class AccessionId extends AuditedVersionedModel implements IdUUID {
this.lists = lists;
}
@IgnoreField
public PDCI getPdci() {
return pdci;
}
......
......@@ -29,14 +29,17 @@ import javax.persistence.Table;
import javax.persistence.Transient;
import org.genesys.blocks.model.VersionedModel;
import org.genesys.custom.elasticsearch.IgnoreField;
@Entity
@Table(name = "pdci", indexes = { @Index(columnList = "score") })
public class