Commit c95547e0 authored by Matija Obreza's avatar Matija Obreza

ES institute.networks, taxonomy.genusSpecies

Drop index if mapping doesn't match (TODO re-index with scroll)
parent 7c325d1a
......@@ -3,7 +3,9 @@ package org.genesys2.server.model.elastic;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
......@@ -17,6 +19,7 @@ import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.AccessionRemark;
import org.genesys2.server.model.genesys.SvalbardData;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.Organization;
import org.springframework.data.annotation.Id;
import org.springframework.data.annotation.Version;
import org.springframework.data.elasticsearch.annotations.Document;
......@@ -45,9 +48,7 @@ public class AccessionDetails {
private String acqSrc;
private Boolean available;
private Date createdDate;
// FIXME @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
// TODO Also handle ElasticsearchSearchServiceImpl and move to #isAnalyzed()
// section!
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private List<String> duplSite;
private Boolean mlsStatus;
private Boolean art15;
......@@ -63,8 +64,7 @@ public class AccessionDetails {
@Field(type = FieldType.Nested)
private ArrayList<Alias> aliases;
private Svalbard sgsv;
// FIXME TODO Enable inSgsv!
// private boolean inSgsv;
private boolean inSgsv;
@Field(type = FieldType.Nested, store = false)
private ArrayList<Remark> remarks;
......@@ -124,8 +124,7 @@ public class AccessionDetails {
public void svalbard(SvalbardData svalbardData) {
this.sgsv = Svalbard.from(svalbardData);
// FIXME TODO inSgsv
// this.inSgsv = this.sgsv != null;
this.inSgsv = this.sgsv != null;
}
public void remarks(List<AccessionRemark> listAccessionRemarks) {
......@@ -394,11 +393,26 @@ public class AccessionDetails {
this.art15 = art15;
}
// TODO FIXME enable #inSgsv
// public boolean isInSgsv() {
// return inSgsv;
// }
// public void setInSgsv(boolean inSgsv) {
// this.inSgsv = inSgsv;
// }
public boolean isInSgsv() {
return inSgsv;
}
public void setInSgsv(boolean inSgsv) {
this.inSgsv = inSgsv;
}
public void networks(List<Organization> organizations) {
if (organizations == null || organizations.isEmpty())
return;
Set<String> networks = new HashSet<String>();
for (Organization org : organizations) {
networks.add(org.getSlug());
}
if (institute == null)
institute = new Institute();
institute.setNetworks(networks);
}
}
......@@ -12,7 +12,9 @@ public class Alias {
private String name;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String lang;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String usedBy;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private AliasType type;
public static Alias from(AccessionAlias aa) {
......
......@@ -8,17 +8,18 @@ import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.genesys2.server.model.genesys.AccessionCollect;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
public class Collect {
public Set<String> collCode;
@Field(searchAnalyzer = "keyword", type = FieldType.String)
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
public String collDate;
public String collInstAddr;
public String collMissId;
public String collName;
@Field(searchAnalyzer = "keyword", type = FieldType.String)
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
public String collNumb;
public String collSite;
public Integer collSrc;
......
......@@ -2,6 +2,9 @@ package org.genesys2.server.model.elastic;
import org.apache.commons.lang.StringUtils;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
public class Geo {
......@@ -11,6 +14,7 @@ public class Geo {
private Double longitude;
private Double[] coordinates;
private Double elevation;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String datum;
public static Geo from(AccessionGeo ag) {
......
package org.genesys2.server.model.elastic;
import java.util.Set;
import org.genesys2.server.model.impl.FaoInstitute;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
......@@ -15,6 +17,8 @@ public class Institute {
private Country country;
@Field(type = FieldType.String)
private String fullName;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private Set<String> networks;
public static Institute from(FaoInstitute institute) {
if (institute == null)
......@@ -59,4 +63,11 @@ public class Institute {
this.fullName = fullName;
}
public Set<String> getNetworks() {
return networks;
}
public void setNetworks(Set<String> networks) {
this.networks = networks;
}
}
\ No newline at end of file
......@@ -13,9 +13,9 @@ public class Taxonomy {
private String genus;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String species;
// TODO FIXME Enable non-indexed access to genus+species
// @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
// private String genusSpecies;
// Enable non-indexed access to genus+species
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String genusSpecies;
private String spAuthor;
private String subtax;
private String subtAuthor;
......@@ -28,6 +28,7 @@ public class Taxonomy {
t.sciName = taxonomy.getTaxonName();
t.genus = taxonomy.getGenus();
t.species = taxonomy.getSpecies();
t.genusSpecies = taxonomy.getGenus() + " " + taxonomy.getSpecies();
t.spAuthor = StringUtils.defaultIfBlank(taxonomy.getSpAuthor(), null);
t.subtax = StringUtils.defaultIfBlank(taxonomy.getSubtaxa(), null);
t.subtAuthor = StringUtils.defaultIfBlank(taxonomy.getSubtAuthor(), null);
......@@ -58,6 +59,14 @@ public class Taxonomy {
this.species = species;
}
public void setGenusSpecies(String genusSpecies) {
this.genusSpecies = genusSpecies;
}
public String getGenusSpecies() {
return genusSpecies;
}
public String getSpAuthor() {
return spAuthor;
}
......
......@@ -441,22 +441,39 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG.info("Initializing index");
elasticsearchTemplate.createIndex(AccessionDetails.class);
LOG.info("Putting mapping");
elasticsearchTemplate.putMapping(AccessionDetails.class);
try {
elasticsearchTemplate.putMapping(AccessionDetails.class);
} catch (Throwable e) {
LOG.error("Mapping mismatch. Need to reindex.");
reindexExistingData(AccessionDetails.class);
}
LOG.info("Refreshing");
elasticsearchTemplate.refresh(AccessionDetails.class, true);
// genesysarchive
Map<?, ?> indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
// Ensure ES index 'genesysarchive'
if (!elasticsearchTemplate.indexExists("genesysarchive")) {
LOG.info("Initializing index");
LOG.info("Initializing genesysarchive");
elasticsearchTemplate.createIndex("genesysarchive");
LOG.info("Copying mapping to genesysarchive");
elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
}
Map<?, ?> indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
// for (Object key : indexMapping.keySet()) {
// System.err.println("Mapping ke=" + key + " val=" +
// indexMapping.get(key));
// }
LOG.info("Copying mapping");
elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
}
// TODO FIXME Don't delete index; use aliases!
private void reindexExistingData(Class<?> clazz) {
LOG.warn("Deleting index for class=" + clazz);
elasticsearchTemplate.deleteIndex(clazz);
// Make new index
LOG.warn("Creating new index for class=" + clazz);
elasticsearchTemplate.createIndex(clazz);
elasticsearchTemplate.putMapping(clazz);
// TODO Scan & Scroll
// TODO Re-alias
// But... For now, don't do anything
}
}
......@@ -36,7 +36,6 @@ import org.genesys2.server.model.filters.BasicFilter;
import org.genesys2.server.model.filters.CodedMethodFilter;
import org.genesys2.server.model.filters.GenesysFilter;
import org.genesys2.server.model.filters.GenesysFilter.DataType;
import org.genesys2.server.model.filters.GenesysFilter.FilterType;
import org.genesys2.server.model.filters.I18nListFilter;
import org.genesys2.server.model.filters.MethodFilter;
import org.genesys2.server.model.filters.ValueName;
......@@ -94,8 +93,8 @@ public class FilterHandler {
this.availableFilters.add(new BasicFilter(FilterConstants.ORGANIZATION, DataType.STRING));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTCODE, "/explore/ac/instCode"));
this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.SGSV, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.MLSSTATUS, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.ART15, DataType.BOOLEAN));
......
......@@ -317,6 +317,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
Accession accession = getAccession(accessionId);
AccessionDetails ad = AccessionDetails.from(accession);
ad.networks(organizationRepository.getOrganizations(accession.getInstitute()));
ad.aliases(listAccessionAliases(accession));
ad.exch(listAccessionExchange(accession));
ad.collect(listAccessionCollect(accession));
......
......@@ -141,7 +141,7 @@ public class ElasticUpdater {
try {
// Sleep to allow {@link ElasticUpdaterProcessor} to finish
Thread.sleep(500);
Thread.sleep(1000);
} catch (InterruptedException e) {
}
......
......@@ -71,15 +71,10 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
if (toRemove != null) {
removeNode(toRemove);
// updatedIndices.add(toRemove.getClassName());
}
if (i % 100 == 0) {
LOG.info("Queue size remove=" + elasticRemoveQueue.size() + " update=" + elasticUpdateQueue.size() + " deleted=" + i);
// if (!updatedIndices.isEmpty()) {
// updatedIndices.clear();
// }
}
} while (running && toRemove != null);
......@@ -95,15 +90,10 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
if (toUpdate != null) {
updateNode(toUpdate);
// updatedIndices.add(toUpdate.getClassName());
}
if (i % 100 == 0) {
LOG.info("Queue size remove=" + elasticRemoveQueue.size() + " update=" + elasticUpdateQueue.size() + " indexed=" + i);
// if (!updatedIndices.isEmpty()) {
// updatedIndices.clear();
// }
}
} while (running && toUpdate != null);
......@@ -122,10 +112,6 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
buckets.clear();
}
// if (!updatedIndices.isEmpty()) {
// updatedIndices.clear();
// }
} catch (HazelcastInstanceNotActiveException e) {
LOG.warn("Hazelcast not active.");
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment