Skip to content

GitLab

Commit ed393ff4 authored by Matija Obreza's avatar Matija Obreza
Browse files

Fixed #45 Allow range filtering on ACCENUMB "sequential number"

parent 760e2158
Hide whitespace changes
Inline Side-by-side
src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java
View file @ ed393ff4
......@@ -40,6 +40,7 @@ public class AccessionDetails {
private UUID uuid;
@Field(type = FieldType.String)
private String acceNumb;
private Float seqNo;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String acqDate;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
......@@ -99,6 +100,7 @@ public class AccessionDetails {
ad.createdDate = accession.getCreatedDate();
ad.modifiedDate = accession.getLastModifiedDate();
ad.acceNumb = accession.getAccessionName();
ad.seqNo = accession.getSeqNo();
ad.acqDate = accession.getAcquisitionDate();
ad.acqSrc = accession.getAcquisitionSource();
ad.available = accession.getAvailability();
......@@ -209,6 +211,14 @@ public class AccessionDetails {
this.acceNumb = acceNumb;
}
public Float getSeqNo() {
return seqNo;
}
public void setSeqNo(Float seqNo) {
this.seqNo = seqNo;
}
public String getAcqDate() {
return acqDate;
}
......
src/main/java/org/genesys2/server/model/genesys/Accession.java
View file @ ed393ff4
......@@ -25,6 +25,7 @@ import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.OrderBy;
import javax.persistence.Table;
......@@ -37,7 +38,7 @@ import javax.persistence.UniqueConstraint;
*
*/
@Entity
@Table(name = "accession", uniqueConstraints = { @UniqueConstraint(name = "UQ_accession_genus_inst", columnNames = { "instituteId", "taxGenus", "acceNumb" }) })
@Table(name = "accession", uniqueConstraints = { @UniqueConstraint(name = "UQ_accession_genus_inst", columnNames = { "instituteId", "taxGenus", "acceNumb" }) }, indexes = { @Index(name = "IX_seqNo", columnList = "seqNo") })
public class Accession extends AccessionData {
/**
*
......
src/main/java/org/genesys2/server/model/genesys/AccessionData.java
View file @ ed393ff4
......@@ -41,6 +41,7 @@ import org.genesys2.server.model.IdUUID;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.util.MCPDUtil;
import org.genesys2.util.NumberUtils;
import org.springframework.data.annotation.CreatedBy;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedBy;
......@@ -88,6 +89,9 @@ public abstract class AccessionData implements IdUUID, Serializable {
@Column(name = "acceNumb", nullable = false, length = 128)
private String accessionName;
@Column(name = "seqNo", nullable = false)
private float seqNo = 0;
// @Cache(usage=CacheConcurrencyStrategy.READ_WRITE)
@ManyToOne(cascade = {}, optional = false)
@JoinColumn(name = "taxonomyId2")
......@@ -148,7 +152,10 @@ public abstract class AccessionData implements IdUUID, Serializable {
this.origin = null;
this.instituteCode = getInstitute().getCode();
this.storage=MCPDUtil.toMcpdArray(getStoRage());
this.storage = MCPDUtil.toMcpdArray(getStoRage());
// Update the ACCENUMB number
this.seqNo = NumberUtils.numericValue(this.accessionName);
}
public FaoInstitute getInstitute() {
......@@ -373,4 +380,12 @@ public abstract class AccessionData implements IdUUID, Serializable {
public void setAcceUrl(String acceUrl) {
this.acceUrl = acceUrl;
}
public float getSeqNo() {
return seqNo;
}
public void setSeqNo(float acceNumbNumb) {
this.seqNo = acceNumbNumb;
}
}
src/main/java/org/genesys2/server/model/genesys/AccessionHistoric.java
View file @ ed393ff4
......@@ -24,6 +24,7 @@ import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.OrderBy;
import javax.persistence.Table;
......@@ -35,7 +36,7 @@ import javax.persistence.Table;
*
*/
@Entity
@Table(name = "accessionhistoric")
@Table(name = "accessionhistoric", indexes = { @Index(name = "IX_seqNo", columnList = "seqNo") })
public class AccessionHistoric extends AccessionData {
/**
......
src/main/java/org/genesys2/server/persistence/domain/GenesysLowlevelRepository.java
View file @ ed393ff4
......@@ -16,6 +16,8 @@
package org.genesys2.server.persistence.domain;
import java.util.List;
import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
......@@ -56,4 +58,6 @@ public interface GenesysLowlevelRepository {
void listAccessionIds(AppliedFilters filter, Sort sort, RowCallbackHandler rowCallbackHandler);
void updateAccessionSequentialNumber(List<Object[]> data);
}
src/main/java/org/genesys2/server/persistence/domain/GenesysLowlevelRepositoryCustomImpl.java
View file @ ed393ff4
......@@ -34,11 +34,13 @@ import org.springframework.cache.annotation.Cacheable;
import org.springframework.data.domain.Sort;
import org.springframework.jdbc.BadSqlGrammarException;
import org.springframework.jdbc.core.ArgumentPreparedStatementSetter;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.PreparedStatementCreator;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
@Repository
......@@ -252,7 +254,6 @@ public class GenesysLowlevelRepositoryCustomImpl implements GenesysLowlevelRepos
}
}, rowCallbackHandler);
}
@Override
public void listAccessionIds(final AppliedFilters filter, final Sort sort, final RowCallbackHandler rowCallbackHandler) {
......@@ -268,14 +269,13 @@ public class GenesysLowlevelRepositoryCustomImpl implements GenesysLowlevelRepos
this.jdbcTemplate.query(new PreparedStatementCreator() {
@Override
public PreparedStatement createPreparedStatement(final Connection con) throws SQLException {
final PreparedStatement stmt = con.prepareStatement(directQuery
.getQuery("a.id"));
final PreparedStatement stmt = con.prepareStatement(directQuery.getQuery("a.id"));
final ArgumentPreparedStatementSetter apss = new ArgumentPreparedStatementSetter(directQuery.getParameters());
apss.setValues(stmt);
// Set mysql JConnector to stream results
// stmt.setFetchSize(Integer.MIN_VALUE);
// stmt.setFetchSize(Integer.MIN_VALUE);
return stmt;
}
}, rowCallbackHandler);
......@@ -405,4 +405,27 @@ public class GenesysLowlevelRepositoryCustomImpl implements GenesysLowlevelRepos
}
}, rowCallbackHandler);
}
/**
* Data[] = accessionId, acceNumb and seqNo
*/
@Override
@Transactional(propagation = Propagation.REQUIRES_NEW, readOnly = false)
public void updateAccessionSequentialNumber(final List<Object[]> data) {
if (data == null || data.size() == 0)
return;
jdbcTemplate.batchUpdate("update accession set seqNo = ? where id = ? and acceNumb = ?", new BatchPreparedStatementSetter() {
public void setValues(PreparedStatement ps, int i) throws SQLException {
Object[] o = data.get(i);
ps.setFloat(1, (float) o[2]);
ps.setLong(2, (long) o[0]);
ps.setString(3, (String) o[1]);
}
public int getBatchSize() {
return data.size();
}
});
}
}
src/main/java/org/genesys2/server/service/ElasticService.java
View file @ ed393ff4
......@@ -44,4 +44,6 @@ public interface ElasticService {
List<String> autocompleteSearch(String query) throws SearchException;
void regenerateAccessionSequentialNumber();
}
src/main/java/org/genesys2/server/service/FilterConstants.java
View file @ ed393ff4
......@@ -71,4 +71,6 @@ public interface FilterConstants {
public static final String TAXONOMY_GENUSSPECIES = "taxonomy.genusSpecies";
public static final String HISTORIC = "historic";
public static final String SEQUENTIAL_NUMBER = "seqNo";
}
src/main/java/org/genesys2/server/service/GenesysService.java
View file @ ed393ff4
......@@ -255,4 +255,6 @@ public interface GenesysService {
PhenoStatistics statisticsPheno(FaoInstitute faoInstitute);
void regenerateAccessionSequentialNumber();
}
src/main/java/org/genesys2/server/service/impl/DirectMysqlQuery.java
View file @ ed393ff4
......@@ -94,6 +94,9 @@ public class DirectMysqlQuery {
}
public DirectMysqlQuery jsonFilter(AppliedFilters filters, MethodResolver methodResolver) {
if (filters == null)
return this;
return join(filters).filter(filters, methodResolver);
}
......@@ -157,6 +160,7 @@ public class DirectMysqlQuery {
createQuery(whereBuffer, "t.taxSpecies", filters.get("speciesId"), params);
}
createQuery(whereBuffer, "a.acceNumb", filters.get(FilterConstants.ACCENUMB), params);
createQuery(whereBuffer, "a.seqNo", filters.get(FilterConstants.SEQUENTIAL_NUMBER), params);
createQuery(whereBuffer, "a.orgCty", filters.get(FilterConstants.ORGCTY_ISO3), params);
createQuery(whereBuffer, "a.instCode", filters.get(FilterConstants.INSTCODE), params);
createQuery(whereBuffer, "faocty.code3", filters.get(FilterConstants.INSTITUTE_COUNTRY_ISO3), params);
......
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
View file @ ed393ff4
package org.genesys2.server.service.impl;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
......@@ -8,6 +10,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.index.query.AndFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.OrFilterBuilder;
......@@ -19,6 +22,7 @@ import org.elasticsearch.search.sort.SortOrder;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.filters.GenesysFilter;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService;
......@@ -32,10 +36,12 @@ import org.genesys2.server.service.impl.FilterHandler.MaxValueFilter;
import org.genesys2.server.service.impl.FilterHandler.MinValueFilter;
import org.genesys2.server.service.impl.FilterHandler.StartsWithFilter;
import org.genesys2.server.service.impl.FilterHandler.ValueRangeFilter;
import org.genesys2.util.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
......@@ -47,6 +53,9 @@ import org.springframework.data.elasticsearch.core.facet.result.TermResult;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.SearchQuery;
import org.springframework.data.elasticsearch.core.query.UpdateQuery;
import org.springframework.data.elasticsearch.core.query.UpdateQueryBuilder;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
......@@ -75,6 +84,10 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
private final Map<String, Class<?>> clazzMap;
@Autowired
@Qualifier("genesysLowlevelRepositoryCustomImpl")
private GenesysLowlevelRepository genesysLowlevelRepository;
public ElasticsearchSearchServiceImpl() {
clazzMap = new HashMap<String, Class<?>>();
clazzMap.put(Accession.class.getName(), AccessionDetails.class);
......@@ -301,6 +314,12 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
if (!clazzMap.containsKey(className)) {
return;
}
// TODO Isn't that handled above?
if (!Accession.class.getName().equals(className)) {
LOG.warn("Unsupported class " + className);
return;
}
if (ids.isEmpty()) {
LOG.info("Skipping empty updateAll.");
return;
......@@ -308,12 +327,8 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
if (LOG.isDebugEnabled())
LOG.debug("Updating " + className + " bulk_size=" + ids.size());
List<IndexQuery> queries = new ArrayList<IndexQuery>();
if (!Accession.class.getName().equals(className)) {
LOG.warn("Unsupported class " + className);
return;
}
List<IndexQuery> queries = new ArrayList<IndexQuery>();
Set<AccessionDetails> ads = genesysService.getAccessionDetails(ids);
// If Accession, update PDCI
......@@ -398,4 +413,69 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
// But... For now, don't do anything
}
@Override
public void regenerateAccessionSequentialNumber() {
final List<Object[]> batch = new ArrayList<Object[]>();
genesysLowlevelRepository.listAccessions(null, new RowCallbackHandler() {
private int counter = 0;
@Override
public void processRow(ResultSet rs) throws SQLException {
long acceId = rs.getLong(1);
String acceNumb = rs.getString(4);
counter++;
Object[] o = new Object[] { acceId, acceNumb, NumberUtils.numericValue(acceNumb) };
batch.add(o);
if (batch.size() >= 100) {
if (counter % 100000 == 0)
LOG.info("Regenerating sequential numbers " + counter);
else
LOG.debug("Regenerating sequential numbers " + counter);
updateAccessionSequentialNumber(batch);
batch.clear();
}
}
});
updateAccessionSequentialNumber(batch);
}
/**
* Data[] = accessionId, acceNumb and seqNo
*
* @param batch
*/
private void updateAccessionSequentialNumber(List<Object[]> batch) {
// TODO Need DATAES-159 for bulkUpdate (available in 1.3.0 M1)
// List<UpdateQuery> queries = new ArrayList<UpdateQuery>();
List<Long> missingDocIds = new ArrayList<Long>(batch.size());
for (Object[] o : batch) {
try {
IndexRequest indexRequest = new IndexRequest();
indexRequest.source("seqNo", o[2]);
UpdateQuery updateQuery = new UpdateQueryBuilder().withClass(AccessionDetails.class).withId(o[0].toString()).withIndexRequest(indexRequest)
.build();
elasticsearchTemplate.update(updateQuery);
// LOG.debug("ES added seqNo to " + o[0].toString());
} catch (Throwable e) {
// LOG.error(e.getMessage());
missingDocIds.add((Long) o[0]);
}
}
if (missingDocIds.size() > 0) {
// LOG.info("Reindexing accession documents: " +
// missingDocIds.size());
updateAll(Accession.class.getName(), missingDocIds);
}
}
}
src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
View file @ ed393ff4
......@@ -102,6 +102,7 @@ import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.spring.SecurityContextUtil;
import org.genesys2.util.NumberUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.cache.annotation.CacheEvict;
......@@ -1733,4 +1734,38 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
}
return stats;
}
@Transactional
@Override
public void regenerateAccessionSequentialNumber() {
final List<Object[]> batch = new ArrayList<Object[]>();
genesysLowlevelRepository.listAccessions(null, new RowCallbackHandler() {
private int counter = 0;
@Override
public void processRow(ResultSet rs) throws SQLException {
long acceId = rs.getLong(1);
String acceNumb = rs.getString(4);
counter++;
Object[] o = new Object[] { acceId, acceNumb, NumberUtils.numericValue(acceNumb) };
batch.add(o);
if (batch.size() >= 100) {
if (counter % 20000 == 0)
LOG.info("Regenerating sequential numbers " + counter);
else
LOG.debug("Regenerating sequential numbers " + counter);
genesysLowlevelRepository.updateAccessionSequentialNumber(batch);
batch.clear();
}
}
});
genesysLowlevelRepository.updateAccessionSequentialNumber(batch);
}
}
src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java
View file @ ed393ff4
......@@ -16,17 +16,36 @@
package org.genesys2.server.servlet.controller.admin;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.service.*;
import org.genesys2.server.service.CountryNamesUpdater;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoRegionService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.impl.ContentSanitizer;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.server.service.worker.*;
import org.genesys2.server.service.worker.ElasticUpdater;
import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.service.worker.SGSVUpdate;
import org.genesys2.server.service.worker.WorldClimUpdater;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.jdbc.core.RowCallbackHandler;
......@@ -38,14 +57,7 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.fasterxml.jackson.databind.ObjectMapper;
@Controller
@RequestMapping("/admin")
......@@ -60,6 +72,8 @@ public class AdminController {
@Autowired
ElasticUpdater elasticUpdater;
@Autowired
ElasticService elasticService;
@Autowired
GenesysService genesysService;
......@@ -367,9 +381,22 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(value = "/georegion", method = RequestMethod.POST)
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "georegion")
public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException {
geoRegionService.updateGeoRegionData();
return "redirect:/admin/";
}
/**
* Scan AccessionData table and convert ACCENUMB to ACCENUMBNUMB (extract
* the number from the ACCNUMB)
*
* @return
*/
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "accenumbnumb")
public String regenerateNumbNumbs() {
genesysService.regenerateAccessionSequentialNumber();
elasticService.regenerateAccessionSequentialNumber();
return "redirect:/admin/";
}
}
src/main/java/org/genesys2/util/NumberUtils.java
View file @ ed393ff4
......@@ -16,6 +16,9 @@
package org.genesys2.util;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
......@@ -23,6 +26,7 @@ import org.apache.commons.logging.LogFactory;
public abstract class NumberUtils {
public static final Log LOG = LogFactory.getLog(NumberUtils.class);
private static Pattern digits = Pattern.compile("\\d+");
/**
* Utility to parse doubles
......@@ -70,4 +74,29 @@ public abstract class NumberUtils {
public static <T> boolean areEqual(T a, T b) {
return a == null && b == null || a != null && a.equals(b) || b != null && b.equals(a);
}
/**
* Extract decimal number from a String.
*/
public static float numericValue(String input) {
float v = 0.0f;
if (StringUtils.isBlank(input))
return v;
Matcher m = digits.matcher(input);
// integer
if (m.find()) {
v += Long.parseLong(m.group());
}
// decimals
if (m.find()) {
float d = Long.parseLong(m.group());
for (int i = m.group().length(); i > 0; i--) {
d /= 10;
}
v += d;
}
return v;
}
}
src/main/webapp/WEB-INF/jsp/admin/index.jsp
View file @ ed393ff4
......@@ -14,11 +14,19 @@
<%@ include file="/WEB-INF/jsp/admin/menu.jsp"%>
<form method="post" action="<c:url value="/admin/assign-uuid" />">
<form method="post" action="<c:url value="/admin/admin-action" />">
<input type="submit" class="btn btn-default" name="accenumbnumb" value="ACCENUMB-NUMB" />
<input type="submit" class="btn btn-default" name="georegion" value="GeoRegion" />
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<form method="post" action="<c:url value="/admin/assign-uuid" />">
<input type="submit" class="btn btn-default" value="Assign missing UUIDs" />
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<form method="post" action="<c:url value="/admin/pdci" />">
<input type="submit" class="btn btn-default" value="Calculate missing PDCI" />