Commit ffec74c0 authored by Matija Obreza's avatar Matija Obreza

Accession#storage to List<Integer>, background scanner

Allow filtering on storage
parent 3cbe0fbb
......@@ -17,11 +17,17 @@
package org.genesys2.server.model.genesys;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OrderBy;
import javax.persistence.Table;
import org.genesys2.server.lucene.genesys.AccessionBridge;
......@@ -102,6 +108,12 @@ public class Accession extends VersionedAuditedModel {
@Column(name = "taxSpecies", nullable = true)
private Long taxSpecies;
@Column(name = "storage", nullable = false)
@ElementCollection(fetch = FetchType.LAZY)
@CollectionTable(name = "accessionstorage", joinColumns = @JoinColumn(name = "accessionId"))
@OrderBy("storage")
private List<Integer> stoRage = new ArrayList<Integer>();
public Accession() {
}
......@@ -258,6 +270,14 @@ public class Accession extends VersionedAuditedModel {
this.taxonomy1 = taxonomy1;
}
public List<Integer> getStoRage() {
return stoRage;
}
public void setStoRage(List<Integer> stoRage) {
this.stoRage = stoRage;
}
@Override
public String toString() {
return MessageFormat.format("Accession id={0,number,#} A={3} inst={1} genus={2}", id, instituteCode, taxGenus, accessionName);
......
......@@ -33,6 +33,9 @@ import org.springframework.data.repository.query.Param;
public interface AccessionRepository extends JpaRepository<Accession, Long> {
@Query("select a.id from Accession a")
public List<Long> listAccessionsIds(Pageable pageable);
@Override
@Query(countQuery = "select count(*) from accession")
List<Accession> findAll();
......
......@@ -112,7 +112,7 @@ public interface GenesysService {
Page<Accession> listAccessionsByTaxSpecies(long taxSpeciesId, Pageable pageable);
Page<Accession> listAccessions(Pageable pageable);
List<Long> listAccessionsIds(Pageable pageable);
Page<Accession> listAccessionsByCrop(Crop crop, Pageable pageable);
......
......@@ -27,6 +27,7 @@ import java.util.UUID;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
......@@ -147,6 +148,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
updated |= updateOrgCty(accession, accnJson.get("orgCty"));
updated |= updateUuid(accession, accnJson.get("uuid"));
updated |= updateRemarks(accession, accnJson.get("remarks"), toSaveRemarks, toRemoveRemarks);
updated |= updateStorage(accession, accnJson);
// TODO Move other setters to methods
......@@ -195,17 +197,6 @@ public class BatchRESTServiceImpl implements BatchRESTService {
}
}
value = accnJson.get("storage");
// MUST BE ARRAY
if (value != null) {
final String storage = arrayToString(toMcpdArray(accnJson, "storage"));
if (!StringUtils.equals(storage, accession.getStorage())) {
accession.setStorage(storage);
updated = true;
}
}
value = accnJson.get("acceName");
if (value != null) {
acceNames.put(accession, toMcpdArray(accnJson, "acceName"));
......@@ -422,6 +413,50 @@ public class BatchRESTServiceImpl implements BatchRESTService {
return toSave.size() > 0 || toSaveColl.size() > 0 || toSaveGeo.size() > 0 || toSaveBreed.size() > 0 || toSaveExch.size() > 0;
}
private boolean updateStorage(Accession accession, ObjectNode accnJson) throws RESTApiDataTypeException {
boolean updated = false;
// MUST BE ARRAY
if (accnJson.has("storage")) {
final String storage = arrayToString(toMcpdArray(accnJson, "storage"));
updateAccessionStorage(accession, storage);
}
return updated;
}
public static boolean updateAccessionStorage(Accession accession, String storage) {
boolean updated = false;
if (!StringUtils.equals(storage, accession.getStorage())) {
accession.setStorage(storage);
updated = true;
}
List<Integer> as = accession.getStoRage();
String[] arr = StringUtils.isBlank(storage) ? ArrayUtils.EMPTY_STRING_ARRAY : storage.split("\\s*[;,]\\s*");
// Sometimes double values are registered: 20;20;40 and the loop below
// kicks in
if (updated || as.size() != arr.length) {
List<Integer> toRemove = new ArrayList<Integer>(as);
for (String storageStr : arr) {
int stor = Integer.parseInt(storageStr);
if (!as.contains(stor)) {
as.add(stor);
} else {
// Cast needed to remove the object
toRemove.remove((Integer) stor);
}
}
as.removeAll(toRemove);
updated = true;
}
return updated;
}
private boolean updateRemarks(Accession accession, JsonNode jsonNode, List<AccessionRemark> toSaveRemarks, List<AccessionRemark> toRemoveRemarks)
throws RESTApiDataTypeException {
if (jsonNode == null || jsonNode.isNull()) {
......
......@@ -98,6 +98,10 @@ public class DirectMysqlQuery {
innerJoin("accessioncollect", "col", "col.accessionId=a.id");
}
if (hasFilter(jsonTree, "storage")) {
innerJoin("accessionstorage", "storage", "storage.accessionId=a.id");
}
return this;
}
......@@ -128,6 +132,7 @@ public class DirectMysqlQuery {
createQuery(whereBuffer, "crop.shortName", jsonTree.get("crop"), params);
createQuery(whereBuffer, "accename.name", jsonTree.get("acceName"), params);
createQuery(whereBuffer, "col.collMissId", jsonTree.get("collMissId"), params);
createQuery(whereBuffer, "storage.storage", jsonTree.get("storage"), params);
for (final Iterator<String> it = jsonTree.fieldNames(); it.hasNext();) {
final String fieldName = it.next();
......
......@@ -128,6 +128,8 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
this.availableFilters.add(new GenesysFilterImpl("inTrust", DataType.BOOLEAN));
this.availableFilters.add(new GenesysFilterImpl("available", DataType.BOOLEAN));
this.availableFilters.add(new GenesysFilterImpl("collMissId", DataType.STRING));
this.availableFilters.add(new GenesysI18nListFilterImpl<Integer>("storage", DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12,
13, 20, 30, 40, 50, 99 }));
}
@Override
......
......@@ -236,12 +236,18 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@Override
public Accession getAccession(AccessionIdentifier3 aid3) {
return accessionRepository.findOne(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
Accession accession = accessionRepository.findOne(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
if (accession != null)
accession.getStoRage().size();
return accession;
}
@Override
public Accession getAccession(String instCode, String acceNumb) {
return accessionRepository.findByInstituteCodeAndAccessionName(instCode, acceNumb);
Accession accession = accessionRepository.findByInstituteCodeAndAccessionName(instCode, acceNumb);
if (accession != null)
accession.getStoRage().size();
return accession;
}
@Override
......@@ -249,12 +255,18 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
if (genus == null) {
throw new NullPointerException("Genus is required to load accession by instCode, acceNumb and genus");
}
return accessionRepository.findOne(instCode, acceNumb, genus);
Accession accession = accessionRepository.findOne(instCode, acceNumb, genus);
if (accession != null)
accession.getStoRage().size();
return accession;
}
@Override
public Accession getAccession(long accessionId) {
return accessionRepository.findOne(accessionId);
Accession accession = accessionRepository.findOne(accessionId);
if (accession != null)
accession.getStoRage().size();
return accession;
}
@Override
......@@ -416,8 +428,8 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
}
@Override
public Page<Accession> listAccessions(Pageable pageable) {
return accessionRepository.findAll(pageable);
public List<Long> listAccessionsIds(Pageable pageable) {
return accessionRepository.listAccessionsIds(pageable);
}
@Override
......@@ -540,7 +552,8 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@Transactional(readOnly = false)
public void saveAccession(Accession... accession) {
for (final Accession a : accession) {
LOG.info("Updating " + a);
if (LOG.isDebugEnabled())
LOG.debug("Updating " + a);
accessionRepository.save(a);
}
}
......@@ -601,13 +614,13 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
public void saveExchange(List<AccessionExchange> all) {
accessionExchangeRepository.save(all);
}
@Override
@Transactional(readOnly = false)
public void saveRemarks(List<AccessionRemark> toSaveRemarks) {
accessionRemarkRepository.save(toSaveRemarks);
}
@Override
@Transactional(readOnly = false)
public void removeRemarks(List<AccessionRemark> toRemoveRemarks) {
......
/**
* Copyright 2014 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.service.worker;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.aspect.AsAdmin;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.impl.BatchRESTServiceImpl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Component;
/**
* Scans Accession#storage records and updates Accession#stoRage
*/
@Component
public class AccessionStorageScanner {
public static final Log LOG = LogFactory.getLog(AccessionStorageScanner.class);
@Autowired
private GenesysService genesysService;
private static final int BATCH_SIZE = 50;
@AsAdmin
public void scanStorage() throws InterruptedException {
int page = 0;
do {
page++;
if (page % 100 == 0) {
LOG.info("Scanning accessions page " + page);
} else {
LOG.debug("Scanning accessions page " + page);
}
List<Long> accns = genesysService.listAccessionsIds(new PageRequest(page, BATCH_SIZE));
List<Accession> toSave = new ArrayList<Accession>(accns.size());
if (accns.size() == 0)
break;
for (long accnId : accns) {
Accession accession = genesysService.getAccession(accnId);
boolean updated = BatchRESTServiceImpl.updateAccessionStorage(accession, accession.getStorage());
if (updated) {
toSave.add(accession);
}
}
try {
if (toSave.size() > 0) {
LOG.info("Updating accession#stoRage for size=" + toSave.size());
genesysService.saveAccession(toSave.toArray(new Accession[] {}));
}
} catch (Throwable e) {
LOG.warn(e.getMessage(), e);
}
Thread.sleep(5);
} while (true);
LOG.info("Done scanning accession#storage");
}
}
......@@ -36,12 +36,14 @@ import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.LuceneIndexer;
import org.genesys2.server.service.MappingService;
import org.genesys2.server.service.impl.ContentSanitizer;
import org.genesys2.server.service.worker.AccessionStorageScanner;
import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.service.worker.SGSVUpdate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.core.task.TaskExecutor;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
......@@ -92,6 +94,12 @@ public class AdminController {
@Autowired
private CacheManager cacheManager;
@Autowired
private TaskExecutor taskExecutor;
@Autowired
protected AccessionStorageScanner accessionStorageScanner;
@RequestMapping("/")
public String root(Model model) {
return "/admin/index";
......@@ -308,14 +316,40 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/scanStorage")
public String scanStorage() {
LOG.info("Updating accession.stoRage");
taskExecutor.execute(new Runnable() {
@Override
public void run() {
Thread t = Thread.currentThread();
int prevPrio = t.getPriority();
String prevName = t.getName();
t.setPriority(Thread.MIN_PRIORITY);
t.setName("background.storageScanner");
try {
accessionStorageScanner.scanStorage();
LOG.info("Updating done");
} catch (InterruptedException e) {
LOG.warn("Storage scanner thread interrupted!");
} finally {
LOG.debug("Resetting thread priority");
t.setPriority(prevPrio);
t.setName(prevName);
}
}
});
LOG.info("Job scheduled");
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/clearTilesCache")
public String clearTilesCache() {
final Cache tileServerCache = cacheManager.getCache("tileserver");
System.err.println("tileServerCache=" + tileServerCache.getNativeCache());
@SuppressWarnings("rawtypes")
final
IMap hazelCache = (IMap) tileServerCache.getNativeCache();
final IMap hazelCache = (IMap) tileServerCache.getNativeCache();
LOG.info("Tiles cache size=" + hazelCache.size());
int count = 0;
......
......@@ -350,6 +350,8 @@ filter.close=Close
filter.remove=Remove filter
filter.autocomplete-placeholder=Type more than 3 characters...
filter.collMissId=Collecting mission ID
filter.storage=Type of Germplasm storage
search.page.title=Full-text Search
search.no-results=No matches found for your query.
......
......@@ -133,9 +133,16 @@
<div class="row">
<div class="col-xs-4"><spring:message code="accession.storage" /></div>
<div class="col-xs-8"><c:forEach items="${accession.storage.split('[;,]')}" var="storage">
<div><spring:message code="accession.storage.${storage}" /></div>
</c:forEach></div>
<div class="col-xs-8">
<c:forEach items="${accession.storage.split('[;,]')}" var="storage">
<div><spring:message code="accession.storage.${storage}" /></div>
</c:forEach>
<%--
<c:forEach items="${accession.stoRage}" var="storage">
<div><spring:message code="accession.storage.${storage}" /></div>
</c:forEach>
--%>
</div>
</div>
<div class="row">
......
......@@ -73,7 +73,11 @@
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
</form>
<form method="post" action="<c:url value="/admin/scanStorage" />">
<input type="submit" class="btn btn-default" class="btn btn-default" value="Scan and convert STORAGE" />
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
</form>
<h3>C&E</h3>
<form method="post" action="<c:url value="/admin/refreshMetadataMethods" />">
<input type="submit" class="btn btn-default" class="btn btn-default" value="Recalculate metadata methods" />
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment