ES institute.networks, taxonomy.genusSpecies

Drop index if mapping doesn't match (TODO re-index with scroll)

src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java

@@ -3,7 +3,9 @@ package org.genesys2.server.model.elastic;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
@@ -17,6 +19,7 @@ import org.genesys2.server.model.genesys.AccessionGeo;
 import org.genesys2.server.model.genesys.AccessionRemark;
 import org.genesys2.server.model.genesys.SvalbardData;
 import org.genesys2.server.model.impl.Crop;
+import org.genesys2.server.model.impl.Organization;
 import org.springframework.data.annotation.Id;
 import org.springframework.data.annotation.Version;
 import org.springframework.data.elasticsearch.annotations.Document;
@@ -45,9 +48,7 @@ public class AccessionDetails {
 	private String acqSrc;
 	private Boolean available;
 	private Date createdDate;
-	// FIXME @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
-	// TODO Also handle ElasticsearchSearchServiceImpl and move to #isAnalyzed()
-	// section!
+	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
 	private List<String> duplSite;
 	private Boolean mlsStatus;
 	private Boolean art15;
@@ -63,8 +64,7 @@ public class AccessionDetails {
 	@Field(type = FieldType.Nested)
 	private ArrayList<Alias> aliases;
 	private Svalbard sgsv;
-	// FIXME TODO Enable inSgsv!
-	// private boolean inSgsv;
+	private boolean inSgsv;
 	@Field(type = FieldType.Nested, store = false)
 	private ArrayList<Remark> remarks;
 
@@ -124,8 +124,7 @@ public class AccessionDetails {
 
 	public void svalbard(SvalbardData svalbardData) {
 		this.sgsv = Svalbard.from(svalbardData);
-		// FIXME TODO inSgsv
-		// this.inSgsv = this.sgsv != null;
+		this.inSgsv = this.sgsv != null;
 	}
 
 	public void remarks(List<AccessionRemark> listAccessionRemarks) {
@@ -394,11 +393,26 @@ public class AccessionDetails {
 		this.art15 = art15;
 	}
 
-	// TODO FIXME enable #inSgsv
-	// public boolean isInSgsv() {
-	// return inSgsv;
-	// }
-	// public void setInSgsv(boolean inSgsv) {
-	// this.inSgsv = inSgsv;
-	// }
+	public boolean isInSgsv() {
+		return inSgsv;
+	}
+
+	public void setInSgsv(boolean inSgsv) {
+		this.inSgsv = inSgsv;
+	}
+
+	public void networks(List<Organization> organizations) {
+		if (organizations == null || organizations.isEmpty())
+			return;
+
+		Set<String> networks = new HashSet<String>();
+		for (Organization org : organizations) {
+			networks.add(org.getSlug());
+		}
+		
+		if (institute == null)
+			institute = new Institute();
+		
+		institute.setNetworks(networks);
+	}
 }

src/main/java/org/genesys2/server/model/elastic/Alias.java

@@ -12,7 +12,9 @@ public class Alias {
 	private String name;
 	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
 	private String lang;
+	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
 	private String usedBy;
+	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
 	private AliasType type;
 
 	public static Alias from(AccessionAlias aa) {

src/main/java/org/genesys2/server/model/elastic/Collect.java

@@ -8,17 +8,18 @@ import java.util.Set;
 import org.apache.commons.lang.StringUtils;
 import org.genesys2.server.model.genesys.AccessionCollect;
 import org.springframework.data.elasticsearch.annotations.Field;
+import org.springframework.data.elasticsearch.annotations.FieldIndex;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 
 public class Collect {
 
 	public Set<String> collCode;
-	@Field(searchAnalyzer = "keyword", type = FieldType.String)
+	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
 	public String collDate;
 	public String collInstAddr;
 	public String collMissId;
 	public String collName;
-	@Field(searchAnalyzer = "keyword", type = FieldType.String)
+	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
 	public String collNumb;
 	public String collSite;
 	public Integer collSrc;

src/main/java/org/genesys2/server/model/elastic/Geo.java

@@ -2,6 +2,9 @@ package org.genesys2.server.model.elastic;
 
 import org.apache.commons.lang.StringUtils;
 import org.genesys2.server.model.genesys.AccessionGeo;
+import org.springframework.data.elasticsearch.annotations.Field;
+import org.springframework.data.elasticsearch.annotations.FieldIndex;
+import org.springframework.data.elasticsearch.annotations.FieldType;
 
 public class Geo {
 
@@ -11,6 +14,7 @@ public class Geo {
 	private Double longitude;
 	private Double[] coordinates;
 	private Double elevation;
+	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
 	private String datum;
 
 	public static Geo from(AccessionGeo ag) {

src/main/java/org/genesys2/server/model/elastic/Institute.java

 package org.genesys2.server.model.elastic;
 
+import java.util.Set;
+
 import org.genesys2.server.model.impl.FaoInstitute;
 import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldIndex;
@@ -15,6 +17,8 @@ public class Institute {
 	private Country country;
 	@Field(type = FieldType.String)
 	private String fullName;
+	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
+	private Set<String> networks;
 
 	public static Institute from(FaoInstitute institute) {
 		if (institute == null)
@@ -59,4 +63,11 @@ public class Institute {
 		this.fullName = fullName;
 	}
 
+	public Set<String> getNetworks() {
+		return networks;
+	}
+	
+	public void setNetworks(Set<String> networks) {
+		this.networks = networks;
+	}
 }
\ No newline at end of file

src/main/java/org/genesys2/server/model/elastic/Taxonomy.java

@@ -13,9 +13,9 @@ public class Taxonomy {
 	private String genus;
 	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
 	private String species;
-	// TODO FIXME Enable non-indexed access to genus+species
-	// @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
-	// private String genusSpecies;
+	// Enable non-indexed access to genus+species
+	@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
+	private String genusSpecies;
 	private String spAuthor;
 	private String subtax;
 	private String subtAuthor;
@@ -28,6 +28,7 @@ public class Taxonomy {
 		t.sciName = taxonomy.getTaxonName();
 		t.genus = taxonomy.getGenus();
 		t.species = taxonomy.getSpecies();
+		t.genusSpecies = taxonomy.getGenus() + " " + taxonomy.getSpecies();
 		t.spAuthor = StringUtils.defaultIfBlank(taxonomy.getSpAuthor(), null);
 		t.subtax = StringUtils.defaultIfBlank(taxonomy.getSubtaxa(), null);
 		t.subtAuthor = StringUtils.defaultIfBlank(taxonomy.getSubtAuthor(), null);
@@ -58,6 +59,14 @@ public class Taxonomy {
 		this.species = species;
 	}
 
+	public void setGenusSpecies(String genusSpecies) {
+		this.genusSpecies = genusSpecies;
+	}
+
+	public String getGenusSpecies() {
+		return genusSpecies;
+	}
+
 	public String getSpAuthor() {
 		return spAuthor;
 	}

src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java

@@ -441,22 +441,39 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 		LOG.info("Initializing index");
 		elasticsearchTemplate.createIndex(AccessionDetails.class);
 		LOG.info("Putting mapping");
-		elasticsearchTemplate.putMapping(AccessionDetails.class);
+		try {
+			elasticsearchTemplate.putMapping(AccessionDetails.class);
+		} catch (Throwable e) {
+			LOG.error("Mapping mismatch. Need to reindex.");
+			reindexExistingData(AccessionDetails.class);
+		}
 		LOG.info("Refreshing");
 		elasticsearchTemplate.refresh(AccessionDetails.class, true);
 
-		// genesysarchive
+		Map<?, ?> indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
+
+		// Ensure ES index 'genesysarchive'
 		if (!elasticsearchTemplate.indexExists("genesysarchive")) {
-			LOG.info("Initializing index");
+			LOG.info("Initializing genesysarchive");
 			elasticsearchTemplate.createIndex("genesysarchive");
+			LOG.info("Copying mapping to genesysarchive");
+			elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
 		}
-		Map<?, ?> indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
-		// for (Object key : indexMapping.keySet()) {
-		// System.err.println("Mapping ke=" + key + " val=" +
-		// indexMapping.get(key));
-		// }
-		LOG.info("Copying mapping");
-		elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
+	}
+
+	// TODO FIXME Don't delete index; use aliases!
+	private void reindexExistingData(Class<?> clazz) {
+		LOG.warn("Deleting index for class=" + clazz);
+		elasticsearchTemplate.deleteIndex(clazz);
+		// Make new index
+		LOG.warn("Creating new index for class=" + clazz);
+		elasticsearchTemplate.createIndex(clazz);
+		elasticsearchTemplate.putMapping(clazz);
+
+		// TODO Scan & Scroll
+		// TODO Re-alias
+
+		// But... For now, don't do anything
 	}
 
 }

src/main/java/org/genesys2/server/service/impl/FilterHandler.java

@@ -36,7 +36,6 @@ import org.genesys2.server.model.filters.BasicFilter;
 import org.genesys2.server.model.filters.CodedMethodFilter;
 import org.genesys2.server.model.filters.GenesysFilter;
 import org.genesys2.server.model.filters.GenesysFilter.DataType;
-import org.genesys2.server.model.filters.GenesysFilter.FilterType;
 import org.genesys2.server.model.filters.I18nListFilter;
 import org.genesys2.server.model.filters.MethodFilter;
 import org.genesys2.server.model.filters.ValueName;
@@ -94,8 +93,8 @@ public class FilterHandler {
 
 		this.availableFilters.add(new BasicFilter(FilterConstants.ORGANIZATION, DataType.STRING));
 		this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTCODE, "/explore/ac/instCode"));
-		this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
-		this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
+		this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING).setAnalyzed(true));
+		this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING).setAnalyzed(true));
 		this.availableFilters.add(new BasicFilter(FilterConstants.SGSV, DataType.BOOLEAN));
 		this.availableFilters.add(new BasicFilter(FilterConstants.MLSSTATUS, DataType.BOOLEAN));
 		this.availableFilters.add(new BasicFilter(FilterConstants.ART15, DataType.BOOLEAN));

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -317,6 +317,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 		Accession accession = getAccession(accessionId);
 
 		AccessionDetails ad = AccessionDetails.from(accession);
+		ad.networks(organizationRepository.getOrganizations(accession.getInstitute()));
 		ad.aliases(listAccessionAliases(accession));
 		ad.exch(listAccessionExchange(accession));
 		ad.collect(listAccessionCollect(accession));

src/main/java/org/genesys2/server/service/worker/ElasticUpdater.java

@@ -141,7 +141,7 @@ public class ElasticUpdater {
 
 		try {
 			// Sleep to allow {@link ElasticUpdaterProcessor} to finish
-			Thread.sleep(500);
+			Thread.sleep(1000);
 		} catch (InterruptedException e) {
 		}
 

src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java

@@ -71,15 +71,10 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
 
 						if (toRemove != null) {
 							removeNode(toRemove);
-							// updatedIndices.add(toRemove.getClassName());
 						}
 
 						if (i % 100 == 0) {
 							LOG.info("Queue size remove=" + elasticRemoveQueue.size() + " update=" + elasticUpdateQueue.size() + " deleted=" + i);
-
-							// if (!updatedIndices.isEmpty()) {
-							// updatedIndices.clear();
-							// }
 						}
 
 					} while (running && toRemove != null);
@@ -95,15 +90,10 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
 
 						if (toUpdate != null) {
 							updateNode(toUpdate);
-							// updatedIndices.add(toUpdate.getClassName());
 						}
 
 						if (i % 100 == 0) {
 							LOG.info("Queue size remove=" + elasticRemoveQueue.size() + " update=" + elasticUpdateQueue.size() + " indexed=" + i);
-
-							// if (!updatedIndices.isEmpty()) {
-							// updatedIndices.clear();
-							// }
 						}
 
 					} while (running && toUpdate != null);
@@ -122,10 +112,6 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
 					buckets.clear();
 				}
 
-				// if (!updatedIndices.isEmpty()) {
-				// updatedIndices.clear();
-				// }
-
 			} catch (HazelcastInstanceNotActiveException e) {
 				LOG.warn("Hazelcast not active.");
 			}