Commit 0f9291d5 authored by Matija Obreza's avatar Matija Obreza

@IgnoreField skips serialization for ES index

- Uses custom introspector and mapping generator
parent 52c6d72b
......@@ -14,10 +14,10 @@ The genesys container can be configured via environment variables (`-e` docker f
## Running genesys2-server with maven
- Clone genesys2-server to your computer
- Clone the project code: `git clone ...`
- Create a blank mysql database
- Configure genesys2-server database connection settings
- Start Jetty with `mvn -Dspring.profiles.active=dev jetty:run`
- Configure genesys2-server database connection settings (see below)
- Start Genesys server with `mvn jetty:run`
### Creating a mysql database
......@@ -31,7 +31,7 @@ CREATE DATABASE genesys DEFAULT CHARSET UTF8;
GRANT ALL ON genesys.* TO 'genesys'@'localhost' IDENTIFIED BY 'pwd';
```
Change the settings in `src/main/resources/spring/spring.properties`:
Change the settings in a new file `src/main/resources/genesys.properties`:
```.properties
# genesys2 with mysql database
......
......@@ -13,16 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.custom.elasticsearch;
package org.genesys.custom.elasticsearch;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.elasticsearch.ElasticsearchException;
import java.io.IOException;
import static org.genesys.custom.elasticsearch.CustomMappingBuilder.*;
import static org.genesys.catalog.custom.elasticsearch.CustomMappingBuilder.buildMapping;
import java.io.IOException;
/**
......
......@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.custom.elasticsearch;
package org.genesys.custom.elasticsearch;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.node.NodeClient;
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.custom.elasticsearch;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Allows for excluding fields from being mapped and sent to ES.
*
* @author Matija Obreza
* @since 1 Aug 2018
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.ANNOTATION_TYPE, ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.FIELD})
@Documented
@Inherited
public @interface IgnoreField {
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.component.elastic;
import org.genesys.custom.elasticsearch.IgnoreField;
import com.fasterxml.jackson.databind.introspect.Annotated;
import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
/**
* Extra controls to what gets serialized to JSON before sending data over to
* ES.
*
* - Ignores @Field(index = no) properties
*
* @author Matija Obreza
* @since 1 Aug 2018
*/
public class ElasticJacksonAnnotationIntrospector extends JacksonAnnotationIntrospector {
private static final long serialVersionUID = 6225410633030301962L;
/**
* Ignore properties where @Fileld(index = no)
*/
@Override
protected boolean _isIgnorable(Annotated a) {
if (isExcludedFromEs(a)) {
return true;
}
return super._isIgnorable(a);
}
@Override
public Boolean hasRequiredMarker(AnnotatedMember m) {
if (isExcludedFromEs(m)) {
return false;
}
return super.hasRequiredMarker(m);
}
private boolean isExcludedFromEs(Annotated a) {
IgnoreField ignoreField = _findAnnotation(a, IgnoreField.class);
if (ignoreField != null) {
// System.err.println("Should not serialize @IgnoreField on " + a);
}
return ignoreField != null;
}
@Override
public Class<?>[] findViews(Annotated a) {
if (isExcludedFromEs(a)) {
return new Class<?>[] { NotIndexed.class };
}
return super.findViews(a);
}
public static class NotIndexed {
}
}
......@@ -49,6 +49,7 @@ import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
import com.fasterxml.jackson.annotation.JsonView;
......@@ -78,6 +79,7 @@ public abstract class AccessionData extends AuditedVersionedModel implements IdU
@JoinColumn(name = "instituteId")
@JsonView({ JsonViews.Minimal.class })
@Field(type = FieldType.Object)
@JsonIgnoreProperties({"settings"})
private FaoInstitute institute;
@Column(name = "acceNumb", nullable = false, length = 128)
......
......@@ -43,6 +43,7 @@ import org.genesys.blocks.auditlog.annotations.Audited;
import org.genesys.blocks.model.AuditedVersionedModel;
import org.genesys.blocks.model.IdUUID;
import org.genesys.blocks.model.JsonViews;
import org.genesys.custom.elasticsearch.IgnoreField;
import org.hibernate.annotations.Type;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
......@@ -83,7 +84,7 @@ public class AccessionId extends AuditedVersionedModel implements IdUUID {
@JoinColumn(name = "pdciId", unique = true)
@JsonIgnoreProperties({ "accession" })
@JsonView({ JsonViews.Internal.class })
@Field(type = FieldType.Object)
@IgnoreField
private PDCI pdci;
@Column(name = "storage", nullable = false)
......@@ -187,6 +188,7 @@ public class AccessionId extends AuditedVersionedModel implements IdUUID {
this.lists = lists;
}
@IgnoreField
public PDCI getPdci() {
return pdci;
}
......
......@@ -29,14 +29,17 @@ import javax.persistence.Table;
import javax.persistence.Transient;
import org.genesys.blocks.model.VersionedModel;
import org.genesys.custom.elasticsearch.IgnoreField;
@Entity
@Table(name = "pdci", indexes = { @Index(columnList = "score") })
public class PDCI extends VersionedModel implements AccessionRelated {
private static final long serialVersionUID = -1312366054528702261L;
@IgnoreField
public static final String[] independentItems = { "genus", "species", "spAuthor", "subTaxa", "subtAuthor", "cropName", "acqDate", "sampStat", "donorCode", "donorNumb",
"otherNumb", "duplSite", "storage", "donorName", "duplInstName", "acceUrl", "mlsStat" };
@IgnoreField
public static final String[] dependentItems = { "origCty", "collSite", "latitude", "longitude", "elevation", "collDate", "bredCode", "ancest", "collSrc", "acceName",
"collNumb", "collCode", "collName" };
......@@ -399,10 +402,12 @@ public class PDCI extends VersionedModel implements AccessionRelated {
return cropName;
}
@IgnoreField
public String[] getIndependentItems() {
return independentItems;
}
@IgnoreField
public String[] getDependentItems() {
return dependentItems;
}
......
......@@ -35,6 +35,7 @@ import javax.persistence.Transient;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.JsonViews;
import org.genesys.custom.elasticsearch.IgnoreField;
import org.hibernate.annotations.Type;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
......@@ -98,6 +99,7 @@ public class Country extends BasicModel {
@Lob
@Type(type = "org.hibernate.type.TextType")
@JsonView({ JsonViews.Internal.class })
@IgnoreField
private String nameL;
@Lob
......@@ -112,6 +114,7 @@ public class Country extends BasicModel {
@ManyToOne(cascade = {}, optional = true)
@JoinColumn(name = "replacedBy")
@JsonView({ JsonViews.Internal.class })
@IgnoreField
private Country replacedBy;
public Country() {
......@@ -165,6 +168,7 @@ public class Country extends BasicModel {
return refnameId;
}
@IgnoreField
public String getNameL() {
return nameL;
}
......@@ -215,6 +219,7 @@ public class Country extends BasicModel {
this.wikiLink = wikiLink;
}
@IgnoreField
public Country getReplacedBy() {
return replacedBy;
}
......
......@@ -40,6 +40,7 @@ import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.EntityId;
import org.genesys.blocks.model.JsonViews;
import org.genesys.blocks.security.model.AclAwareModel;
import org.genesys.custom.elasticsearch.IgnoreField;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
......@@ -56,7 +57,7 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
private static final int LEN_ACRONYM = 50;
@Column(unique = true, nullable = false, length = 10)
@Field(index=FieldIndex.not_analyzed)
@Field(index = FieldIndex.not_analyzed)
private String code;
@Column(length = 300)
......@@ -64,7 +65,7 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
@Column(length = 100)
@JsonView({ JsonViews.Public.class })
@Field(index=FieldIndex.not_analyzed)
@Field(index = FieldIndex.not_analyzed)
private String type;
@Column(length = 300)
......@@ -97,7 +98,7 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
@OneToMany(cascade = {})
@JoinColumn(referencedColumnName = "code", name = "instCode")
@MapKey(name = "setting")
@Field(index = FieldIndex.no)
@IgnoreField
private Map<String, FaoInstituteSetting> settings = new HashMap<String, FaoInstituteSetting>();
private long accessionCount;
......@@ -224,6 +225,7 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
this.elevation = elevation;
}
@IgnoreField
public Map<String, FaoInstituteSetting> getSettings() {
return settings;
}
......
......@@ -19,6 +19,7 @@ import javax.persistence.Transient;
import org.genesys.blocks.model.JsonViews;
import org.genesys.blocks.model.VersionedModel;
import org.genesys.custom.elasticsearch.IgnoreField;
import org.hibernate.annotations.Type;
import com.fasterxml.jackson.annotation.JsonIdentityReference;
......@@ -44,11 +45,13 @@ public class GeoRegion extends VersionedModel {
@JoinColumn(name = "parentId")
@JsonView({ JsonViews.Public.class })
@JsonIdentityReference(alwaysAsId = true)
@IgnoreField
private GeoRegion parentRegion;
@OneToMany(fetch = FetchType.LAZY, mappedBy = "region")
@JsonView({ JsonViews.Public.class })
@JsonIdentityReference(alwaysAsId = true)
@IgnoreField
private List<Country> countries;
@Lob
......@@ -76,6 +79,7 @@ public class GeoRegion extends VersionedModel {
this.name = name;
}
@IgnoreField
public List<Country> getCountries() {
return countries;
}
......@@ -84,6 +88,7 @@ public class GeoRegion extends VersionedModel {
this.countries = countries;
}
@IgnoreField
public GeoRegion getParentRegion() {
return parentRegion;
}
......
......@@ -59,7 +59,7 @@ import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.Publishable;
import org.genesys.blocks.model.VersionedModel;
import org.genesys.blocks.model.filters.BasicModelFilter;
import org.genesys.catalog.custom.elasticsearch.CustomMapping;
import org.genesys.custom.elasticsearch.CustomMapping;
import org.genesys2.server.component.elastic.ElasticQueryBuilder;
import org.genesys2.server.service.ElasticsearchService;
import org.slf4j.Logger;
......@@ -139,7 +139,11 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
} else {
LOG.info("Updating write index {} for {}", writeIndex, clazz.getName());
Object mapping = CustomMapping.springDataMapping(clazz, COMMON_TYPE_NAME);
elasticsearchTemplate.putMapping(writeIndex, COMMON_TYPE_NAME, mapping);
try {
elasticsearchTemplate.putMapping(writeIndex, COMMON_TYPE_NAME, mapping);
} catch (Throwable e) {
LOG.error("The ES mapping is no longer compatible for index={} of {}. Please regenerate.", writeIndex, clazz);
}
}
}
} else {
......
......@@ -20,14 +20,15 @@ import java.io.IOException;
import org.elasticsearch.client.Client;
import org.genesys.blocks.model.JsonViews;
import org.genesys.catalog.custom.elasticsearch.EmbeddedClientFactoryBean;
import org.genesys.catalog.model.Partner;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.model.traits.Descriptor;
import org.genesys.catalog.model.traits.DescriptorList;
import org.genesys.catalog.model.vocab.ControlledVocabulary;
import org.genesys.catalog.model.vocab.VocabularyTerm;
import org.genesys.custom.elasticsearch.EmbeddedClientFactoryBean;
import org.genesys2.server.component.elastic.ElasticJPAListener;
import org.genesys2.server.component.elastic.ElasticJacksonAnnotationIntrospector;
import org.genesys2.server.component.elastic.ElasticReindex;
import org.genesys2.server.component.elastic.ElasticReindexProcessor;
import org.genesys2.server.model.genesys.Accession;
......@@ -48,6 +49,7 @@ import org.springframework.data.elasticsearch.client.TransportClientFactoryBean;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.EntityMapper;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
......@@ -159,7 +161,7 @@ public class ElasticsearchConfig {
*/
@Bean
public ElasticsearchTemplate elasticsearchTemplate(final Client client) {
return new ElasticsearchTemplate(client, entityMapper());
return new ElasticsearchTemplate(client, new GenesysEntityMapper());
}
/**
......@@ -182,14 +184,10 @@ public class ElasticsearchConfig {
return new ElasticReindexProcessor();
}
private EntityMapper entityMapper() {
return new GenesysEntityMapper();
}
/**
* The Class GenesysEntityMapper.
*/
public class GenesysEntityMapper implements EntityMapper {
public static class GenesysEntityMapper implements EntityMapper {
private final ObjectMapper mapper;
private ObjectWriter writer;
......@@ -204,6 +202,8 @@ public class ElasticsearchConfig {
mapper.registerModule(hib4);
// deserialization
mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
mapper.enable(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT);
mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
// ignore stuff we don't understand
......@@ -216,14 +216,14 @@ public class ElasticsearchConfig {
mapper.disable(SerializationFeature.EAGER_SERIALIZER_FETCH);
// explicit json views: every fields needs to be annotated, therefore enabled
mapper.enable(MapperFeature.DEFAULT_VIEW_INCLUSION);
mapper.setAnnotationIntrospector(new ElasticJacksonAnnotationIntrospector());
mapper.writerWithView(JsonViews.Indexed.class);
writer = mapper.writerWithDefaultPrettyPrinter();
}
@Override
public String mapToString(final Object object) throws IOException {
String x = writer.writeValueAsString(object);
// System.err.println(x);
String x = mapper.writeValueAsString(object);
// System.err.println(x);
return x;
}
......
......@@ -15,16 +15,16 @@
*/
package org.genesys.test.config;
import java.io.IOException;
import org.elasticsearch.client.Client;
import org.genesys.catalog.custom.elasticsearch.EmbeddedClientFactoryBean;
import org.genesys.catalog.model.Partner;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.model.traits.Descriptor;
import org.genesys.catalog.model.traits.DescriptorList;
import org.genesys.catalog.model.vocab.ControlledVocabulary;
import org.genesys.catalog.model.vocab.VocabularyTerm;
import org.genesys.custom.elasticsearch.EmbeddedClientFactoryBean;
import org.genesys2.server.component.elastic.ElasticJPAListener;
import org.genesys2.server.component.elastic.ElasticReindexProcessor;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.ActivityPost;
import org.genesys2.server.model.impl.Article;
......@@ -33,16 +33,12 @@ import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.impl.ElasticsearchServiceImpl;
import org.genesys2.spring.config.ElasticsearchConfig;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.EntityMapper;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
/**
* ElasticsearchConfig for JUnit
......@@ -65,15 +61,15 @@ public class TestElasticsearchConfig implements InitializingBean {
this.dataPath = "target/test-es/" + System.currentTimeMillis();
}
// @Bean
// public ElasticJPAListener listener() {
// return new ElasticJPAListener();
// }
//
// @Bean
// public ElasticReindexProcessor reindexProcessor() {
// return new ElasticReindexProcessor();
// }
@Bean
public ElasticJPAListener listener() {
return new ElasticJPAListener();
}
@Bean
public ElasticReindexProcessor reindexProcessor() {
return new ElasticReindexProcessor();
}
/**
* Node client.
......@@ -123,54 +119,9 @@ public class TestElasticsearchConfig implements InitializingBean {
* @return the elasticsearch template
*/
@Bean
public ElasticsearchTemplate elasticsearchTemplate(final Client client, final EntityMapper entityMapper) {
final ElasticsearchTemplate t = new ElasticsearchTemplate(client, entityMapper);
public ElasticsearchTemplate elasticsearchTemplate(final Client client) {
final ElasticsearchTemplate t = new ElasticsearchTemplate(client, new ElasticsearchConfig.GenesysEntityMapper());
return t;
}
/**
* Entity mapper.
*
* @return the entity mapper
*/
@Bean
public EntityMapper entityMapper() {
return new GenesysEntityMapper();
}
/**
* The Class GenesysEntityMapper.
*/
public class GenesysEntityMapper implements EntityMapper {
/** The object mapper. */
private final ObjectMapper objectMapper;
/**
* Instantiates a new genesys entity mapper.
*/
public GenesysEntityMapper() {
objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
objectMapper.setSerializationInclusion(Include.NON_EMPTY);
}
/* (non-Javadoc)
* @see org.springframework.data.elasticsearch.core.EntityMapper#mapToString(java.lang.Object)
*/
@Override
public String mapToString(final Object object) throws IOException {
return objectMapper.writeValueAsString(object);
}
/* (non-Javadoc)
* @see org.springframework.data.elasticsearch.core.EntityMapper#mapToObject(java.lang.String, java.lang.Class)
*/
@Override
public <T> T mapToObject(final String source, final Class<T> clazz) throws IOException {
return objectMapper.readValue(source, clazz);
}
}
}
package org.genesys.test.simpletest;
import java.io.IOException;
import org.genesys2.server.model.impl.Country;
import org.genesys2.spring.config.ElasticsearchConfig;
import org.genesys2.spring.config.ElasticsearchConfig.GenesysEntityMapper;
import org.junit.Test;
public class ElasticSerializerTest {
GenesysEntityMapper mapper = new ElasticsearchConfig.GenesysEntityMapper();
@Test
public void test1() throws IOException {
Country c = new Country();
c.setCode2("00");
c.setCode3("000");
c.setName("Foo");
c.setNameL("NameL");
System.out.println(mapper.mapToString(c));
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment