Commit e184ca11 authored by Matija Obreza's avatar Matija Obreza

WIP: Extra properties from arrays and referenced schemas

parent ac89e063
...@@ -58,6 +58,7 @@ import org.springframework.data.elasticsearch.annotations.Field; ...@@ -58,6 +58,7 @@ import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.FieldType;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonView; import com.fasterxml.jackson.annotation.JsonView;
/** /**
...@@ -110,6 +111,7 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Acl ...@@ -110,6 +111,7 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Acl
@OrderColumn(name = "position") @OrderColumn(name = "position")
@Field(type = FieldType.Object) @Field(type = FieldType.Object)
@JsonView({ JsonViews.Public.class }) @JsonView({ JsonViews.Public.class })
@JsonIgnoreProperties({ "folder" })
private List<RepositoryFile> repositoryFiles; private List<RepositoryFile> repositoryFiles;
/** The creators. */ /** The creators. */
......
...@@ -189,6 +189,7 @@ public class Descriptor extends UuidModel implements SelfCleaning, Publishable, ...@@ -189,6 +189,7 @@ public class Descriptor extends UuidModel implements SelfCleaning, Publishable,
@OrderColumn(name = "idx") @OrderColumn(name = "idx")
@JsonView({ JsonViews.Public.class }) @JsonView({ JsonViews.Public.class })
@Field(type = FieldType.Nested) @Field(type = FieldType.Nested)
@JsonIgnoreProperties(value = { "descriptor" })
private List<VocabularyTerm> terms; private List<VocabularyTerm> terms;
/** The owner. */ /** The owner. */
......
...@@ -36,10 +36,6 @@ import javax.persistence.criteria.CriteriaBuilder; ...@@ -36,10 +36,6 @@ import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Root; import javax.persistence.criteria.Root;
import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator;
import com.fasterxml.jackson.module.jsonSchema.types.ObjectSchema;
import com.google.common.collect.Sets;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.ListUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
...@@ -104,6 +100,13 @@ import org.springframework.transaction.annotation.Transactional; ...@@ -104,6 +100,13 @@ import org.springframework.transaction.annotation.Transactional;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator;
import com.fasterxml.jackson.module.jsonSchema.types.ArraySchema;
import com.fasterxml.jackson.module.jsonSchema.types.ArraySchema.Items;
import com.fasterxml.jackson.module.jsonSchema.types.ObjectSchema;
import com.fasterxml.jackson.module.jsonSchema.types.ReferenceSchema;
import com.google.common.collect.Sets;
import com.querydsl.core.types.EntityPath; import com.querydsl.core.types.EntityPath;
import com.querydsl.core.types.dsl.PathBuilder; import com.querydsl.core.types.dsl.PathBuilder;
import com.querydsl.core.types.dsl.PathBuilderFactory; import com.querydsl.core.types.dsl.PathBuilderFactory;
...@@ -175,8 +178,10 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ ...@@ -175,8 +178,10 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(mapper); JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(mapper);
for (Class<? extends BasicModel> clazz: indexedEntities) { for (Class<? extends BasicModel> clazz: indexedEntities) {
try { try {
JsonSchema schema = schemaGen.generateSchema(clazz); Map<String, ObjectSchema> seenSchemas = new HashMap<>();
jsonSchemas.put(clazz, buildJsonPaths(((ObjectSchema) schema).getProperties(), null)); ObjectSchema schema = (ObjectSchema) schemaGen.generateSchema(clazz);
System.err.println("\n\n\n" + clazz.getName() + "\n\n" + mapper.writeValueAsString(schema));
jsonSchemas.put(clazz, buildJsonPaths(schema, seenSchemas, null));
} catch (Throwable e) { } catch (Throwable e) {
LOG.error("The list of all {} fields is not created.", clazz.getSimpleName(), e); LOG.error("The list of all {} fields is not created.", clazz.getSimpleName(), e);
} }
...@@ -187,24 +192,50 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ ...@@ -187,24 +192,50 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
* Makes a list of all JSON paths for indexed entity and all related types. * Makes a list of all JSON paths for indexed entity and all related types.
* *
* @param properties properties * @param properties properties
* @param seenSchemas
* @param parentPath parentPath * @param parentPath parentPath
*/ */
private Set<String> buildJsonPaths(Map<String, JsonSchema> properties, String parentPath) { private Set<String> buildJsonPaths(ObjectSchema schema, Map<String, ObjectSchema> seenSchemas, String parentPath) {
seenSchemas.put(schema.getId(), schema);
Map<String, JsonSchema> properties = schema.getProperties();
if (MapUtils.isEmpty(properties)) { if (MapUtils.isEmpty(properties)) {
return Collections.emptySet(); return Collections.emptySet();
} }
Set<String> fieldList = new HashSet<>(); Set<String> fieldList = new HashSet<>();
Set<String> keys = properties.keySet(); Set<String> keys = properties.keySet();
keys.removeAll(Sets.newHashSet("_class", "_permissions")); // Remove fields we don't want to check: audit info, generated fields
keys.removeAll(Sets.newHashSet("_class", "_permissions", "createdBy", "createdDate", "lastModifiedBy", "lastModifiedDate", "version"));
for (String path: keys) { for (String path: keys) {
JsonSchema schema = properties.get(path); JsonSchema fieldSchema = properties.get(path);
String fullPath = StringUtils.isBlank(parentPath) ? path : parentPath + "." + path; String fullPath = StringUtils.isBlank(parentPath) ? path : parentPath + "." + path;
if (schema instanceof ObjectSchema) { if (fieldSchema instanceof ObjectSchema) {
fieldList.addAll(buildJsonPaths(((ObjectSchema)schema).getProperties(), fullPath)); fieldList.addAll(buildJsonPaths((ObjectSchema) fieldSchema, seenSchemas, fullPath));
} else if (fieldSchema instanceof ReferenceSchema) {
ObjectSchema referencedSchema = seenSchemas.get(((ReferenceSchema)fieldSchema).get$ref());
fieldList.addAll(buildJsonPaths(referencedSchema, seenSchemas, fullPath));
} else if (fieldSchema instanceof ArraySchema) {
Items items = ((ArraySchema)fieldSchema).getItems();
if (items != null && items.isSingleItems()) {
JsonSchema itemsSchema = items.asSingleItems().getSchema();
if (itemsSchema.isObjectSchema()) {
fieldList.addAll(buildJsonPaths((ObjectSchema) itemsSchema, seenSchemas, fullPath));
} else if (itemsSchema instanceof ReferenceSchema) {
ObjectSchema referencedSchema = seenSchemas.get(((ReferenceSchema)itemsSchema).get$ref());
System.err.println("ref[]: " + fullPath + " " + fieldSchema.getClass().getName());
fieldList.addAll(buildJsonPaths(referencedSchema, seenSchemas, fullPath));
} else {
System.err.println("3: " + fullPath + " " + fieldSchema.getClass().getName());
fieldList.add(fullPath);
}
} else {
System.err.println("2: " + fullPath + " " + fieldSchema.getClass().getName());
fieldList.add(fullPath);
}
} else { } else {
System.err.println(fullPath + " " + fieldSchema.getClass().getName());
fieldList.add(fullPath); fieldList.add(fullPath);
} }
} }
...@@ -808,9 +839,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ ...@@ -808,9 +839,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
LOG.debug("Counting missing values took {}s", response.getTook().getSeconds()); LOG.debug("Counting missing values took {}s", response.getTook().getSeconds());
for (Aggregation agg: response.getAggregations()) { for (Aggregation agg: response.getAggregations()) {
long missingCount = ((InternalMissing) agg).getDocCount(); long missingCount = ((InternalMissing) agg).getDocCount();
if (missingCount > 0) { results.put(agg.getName(), missingCount);
results.put(agg.getName(), missingCount);
}
} }
results.put("_totalCount", response.getHits().getTotalHits()); results.put("_totalCount", response.getHits().getTotalHits());
return results; return results;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment