WIP: Extra properties from arrays and referenced schemas

src/main/java/org/genesys/catalog/model/dataset/Dataset.java

@@ -58,6 +58,7 @@ import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonView;
 
 /**
@@ -110,6 +111,7 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Acl
 	@OrderColumn(name = "position")
 	@Field(type = FieldType.Object)
 	@JsonView({ JsonViews.Public.class })
+	@JsonIgnoreProperties({ "folder" })
 	private List<RepositoryFile> repositoryFiles;
 
 	/** The creators. */

src/main/java/org/genesys/catalog/model/traits/Descriptor.java

@@ -189,6 +189,7 @@ public class Descriptor extends UuidModel implements SelfCleaning, Publishable,
 	@OrderColumn(name = "idx")
 	@JsonView({ JsonViews.Public.class })
 	@Field(type = FieldType.Nested)
+	@JsonIgnoreProperties(value = { "descriptor" })
 	private List<VocabularyTerm> terms;
 
 	/** The owner. */

src/main/java/org/genesys2/server/service/impl/ElasticsearchServiceImpl.java

@@ -36,10 +36,6 @@ import javax.persistence.criteria.CriteriaBuilder;
 import javax.persistence.criteria.CriteriaQuery;
 import javax.persistence.criteria.Root;
 
-import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
-import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator;
-import com.fasterxml.jackson.module.jsonSchema.types.ObjectSchema;
-import com.google.common.collect.Sets;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.collections4.MapUtils;
@@ -104,6 +100,13 @@ import org.springframework.transaction.annotation.Transactional;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
+import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator;
+import com.fasterxml.jackson.module.jsonSchema.types.ArraySchema;
+import com.fasterxml.jackson.module.jsonSchema.types.ArraySchema.Items;
+import com.fasterxml.jackson.module.jsonSchema.types.ObjectSchema;
+import com.fasterxml.jackson.module.jsonSchema.types.ReferenceSchema;
+import com.google.common.collect.Sets;
 import com.querydsl.core.types.EntityPath;
 import com.querydsl.core.types.dsl.PathBuilder;
 import com.querydsl.core.types.dsl.PathBuilderFactory;
@@ -175,8 +178,10 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
 		JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(mapper);
 		for (Class<? extends BasicModel> clazz: indexedEntities) {
 			try {
-				JsonSchema schema = schemaGen.generateSchema(clazz);
-				jsonSchemas.put(clazz, buildJsonPaths(((ObjectSchema) schema).getProperties(), null));
+				Map<String, ObjectSchema> seenSchemas = new HashMap<>();
+				ObjectSchema schema = (ObjectSchema) schemaGen.generateSchema(clazz);
+				 System.err.println("\n\n\n" + clazz.getName() + "\n\n" + mapper.writeValueAsString(schema));
+				jsonSchemas.put(clazz, buildJsonPaths(schema, seenSchemas, null));
 			} catch (Throwable e) {
 				LOG.error("The list of all {} fields is not created.", clazz.getSimpleName(), e);
 			}
@@ -187,24 +192,50 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
 	 * Makes a list of all JSON paths for indexed entity and all related types.
 	 *
 	 * @param properties properties
+	 * @param seenSchemas 
 	 * @param parentPath parentPath
 	 */
-	private Set<String> buildJsonPaths(Map<String, JsonSchema> properties, String parentPath) {
+	private Set<String> buildJsonPaths(ObjectSchema schema, Map<String, ObjectSchema> seenSchemas, String parentPath) {
+		seenSchemas.put(schema.getId(), schema);
+		Map<String, JsonSchema> properties = schema.getProperties();
 		if (MapUtils.isEmpty(properties)) {
 			return Collections.emptySet();
 		}
 
 		Set<String> fieldList = new HashSet<>();
 		Set<String> keys = properties.keySet();
-		keys.removeAll(Sets.newHashSet("_class", "_permissions"));
+		// Remove fields we don't want to check: audit info, generated fields
+		keys.removeAll(Sets.newHashSet("_class", "_permissions", "createdBy", "createdDate", "lastModifiedBy", "lastModifiedDate", "version"));
 
 		for (String path: keys) {
-			JsonSchema schema = properties.get(path);
+			JsonSchema fieldSchema = properties.get(path);
 			String fullPath = StringUtils.isBlank(parentPath) ? path : parentPath + "." + path;
 
-			if (schema instanceof ObjectSchema) {
-				fieldList.addAll(buildJsonPaths(((ObjectSchema)schema).getProperties(), fullPath));
+			if (fieldSchema instanceof ObjectSchema) {
+				fieldList.addAll(buildJsonPaths((ObjectSchema) fieldSchema, seenSchemas, fullPath));
+			} else if (fieldSchema instanceof ReferenceSchema) {
+				ObjectSchema referencedSchema = seenSchemas.get(((ReferenceSchema)fieldSchema).get$ref());
+				fieldList.addAll(buildJsonPaths(referencedSchema, seenSchemas, fullPath));
+			} else if (fieldSchema instanceof ArraySchema) {
+				Items items = ((ArraySchema)fieldSchema).getItems();
+				if (items != null && items.isSingleItems()) {
+					JsonSchema itemsSchema = items.asSingleItems().getSchema();
+					if (itemsSchema.isObjectSchema()) {
+						fieldList.addAll(buildJsonPaths((ObjectSchema) itemsSchema, seenSchemas, fullPath));
+					} else if (itemsSchema instanceof ReferenceSchema) {
+						ObjectSchema referencedSchema = seenSchemas.get(((ReferenceSchema)itemsSchema).get$ref());
+						System.err.println("ref[]: " + fullPath + " " + fieldSchema.getClass().getName());
+						fieldList.addAll(buildJsonPaths(referencedSchema, seenSchemas, fullPath));
+					} else {
+						System.err.println("3: " + fullPath + " " + fieldSchema.getClass().getName());
+						fieldList.add(fullPath);
+					}
+				} else {
+					System.err.println("2: " + fullPath + " " + fieldSchema.getClass().getName());
+					fieldList.add(fullPath);
+				}
 			} else {
+				System.err.println(fullPath + " " + fieldSchema.getClass().getName());
 				fieldList.add(fullPath);
 			}
 		}
@@ -808,9 +839,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
 		LOG.debug("Counting missing values took {}s", response.getTook().getSeconds());
 		for (Aggregation agg: response.getAggregations()) {
 			long missingCount = ((InternalMissing) agg).getDocCount();
-			if (missingCount > 0) {
-				results.put(agg.getName(), missingCount);
-			}
+			results.put(agg.getName(), missingCount);
 		}
 		results.put("_totalCount", response.getHits().getTotalHits());
 		return results;