Accession#storage to List<Integer>, background scanner

Allow filtering on storage

src/main/java/org/genesys2/server/model/genesys/Accession.java

@@ -17,11 +17,17 @@
 package org.genesys2.server.model.genesys;
 
 import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.List;
 
+import javax.persistence.CollectionTable;
 import javax.persistence.Column;
+import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
+import javax.persistence.FetchType;
 import javax.persistence.JoinColumn;
 import javax.persistence.ManyToOne;
+import javax.persistence.OrderBy;
 import javax.persistence.Table;
 
 import org.genesys2.server.lucene.genesys.AccessionBridge;
@@ -102,6 +108,12 @@ public class Accession extends VersionedAuditedModel {
 	@Column(name = "taxSpecies", nullable = true)
 	private Long taxSpecies;
 
+	@Column(name = "storage", nullable = false)
+	@ElementCollection(fetch = FetchType.LAZY)
+	@CollectionTable(name = "accessionstorage", joinColumns = @JoinColumn(name = "accessionId"))
+	@OrderBy("storage")
+	private List<Integer> stoRage = new ArrayList<Integer>();
+
 	public Accession() {
 	}
 
@@ -258,6 +270,14 @@ public class Accession extends VersionedAuditedModel {
 		this.taxonomy1 = taxonomy1;
 	}
 
+	public List<Integer> getStoRage() {
+		return stoRage;
+	}
+
+	public void setStoRage(List<Integer> stoRage) {
+		this.stoRage = stoRage;
+	}
+
 	@Override
 	public String toString() {
 		return MessageFormat.format("Accession id={0,number,#} A={3} inst={1} genus={2}", id, instituteCode, taxGenus, accessionName);

src/main/java/org/genesys2/server/persistence/domain/AccessionRepository.java

@@ -33,6 +33,9 @@ import org.springframework.data.repository.query.Param;
 
 public interface AccessionRepository extends JpaRepository<Accession, Long> {
 
+	@Query("select a.id from Accession a")
+	public List<Long> listAccessionsIds(Pageable pageable);
+
 	@Override
 	@Query(countQuery = "select count(*) from accession")
 	List<Accession> findAll();

src/main/java/org/genesys2/server/service/GenesysService.java

@@ -112,7 +112,7 @@ public interface GenesysService {
 
 	Page<Accession> listAccessionsByTaxSpecies(long taxSpeciesId, Pageable pageable);
 
-	Page<Accession> listAccessions(Pageable pageable);
+	List<Long> listAccessionsIds(Pageable pageable);
 
 	Page<Accession> listAccessionsByCrop(Crop crop, Pageable pageable);
 

src/main/java/org/genesys2/server/service/impl/BatchRESTServiceImpl.java

@@ -27,6 +27,7 @@ import java.util.UUID;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.Predicate;
+import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -147,6 +148,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
 			updated |= updateOrgCty(accession, accnJson.get("orgCty"));
 			updated |= updateUuid(accession, accnJson.get("uuid"));
 			updated |= updateRemarks(accession, accnJson.get("remarks"), toSaveRemarks, toRemoveRemarks);
+			updated |= updateStorage(accession, accnJson);
 
 			// TODO Move other setters to methods
 
@@ -195,17 +197,6 @@ public class BatchRESTServiceImpl implements BatchRESTService {
 				}
 			}
 
-			value = accnJson.get("storage");
-			// MUST BE ARRAY
-			if (value != null) {
-				final String storage = arrayToString(toMcpdArray(accnJson, "storage"));
-
-				if (!StringUtils.equals(storage, accession.getStorage())) {
-					accession.setStorage(storage);
-					updated = true;
-				}
-			}
-
 			value = accnJson.get("acceName");
 			if (value != null) {
 				acceNames.put(accession, toMcpdArray(accnJson, "acceName"));
@@ -422,6 +413,50 @@ public class BatchRESTServiceImpl implements BatchRESTService {
 		return toSave.size() > 0 || toSaveColl.size() > 0 || toSaveGeo.size() > 0 || toSaveBreed.size() > 0 || toSaveExch.size() > 0;
 	}
 
+	private boolean updateStorage(Accession accession, ObjectNode accnJson) throws RESTApiDataTypeException {
+		boolean updated = false;
+
+		// MUST BE ARRAY
+		if (accnJson.has("storage")) {
+			final String storage = arrayToString(toMcpdArray(accnJson, "storage"));
+			updateAccessionStorage(accession, storage);
+		}
+
+		return updated;
+	}
+
+	public static boolean updateAccessionStorage(Accession accession, String storage) {
+		boolean updated = false;
+		if (!StringUtils.equals(storage, accession.getStorage())) {
+			accession.setStorage(storage);
+			updated = true;
+		}
+
+		List<Integer> as = accession.getStoRage();
+		String[] arr = StringUtils.isBlank(storage) ? ArrayUtils.EMPTY_STRING_ARRAY : storage.split("\\s*[;,]\\s*");
+
+		// Sometimes double values are registered: 20;20;40 and the loop below
+		// kicks in
+		if (updated || as.size() != arr.length) {
+			List<Integer> toRemove = new ArrayList<Integer>(as);
+
+			for (String storageStr : arr) {
+				int stor = Integer.parseInt(storageStr);
+				if (!as.contains(stor)) {
+					as.add(stor);
+				} else {
+					// Cast needed to remove the object
+					toRemove.remove((Integer) stor);
+				}
+			}
+
+			as.removeAll(toRemove);
+			updated = true;
+		}
+
+		return updated;
+	}
+
 	private boolean updateRemarks(Accession accession, JsonNode jsonNode, List<AccessionRemark> toSaveRemarks, List<AccessionRemark> toRemoveRemarks)
 			throws RESTApiDataTypeException {
 		if (jsonNode == null || jsonNode.isNull()) {

src/main/java/org/genesys2/server/service/impl/DirectMysqlQuery.java

@@ -98,6 +98,10 @@ public class DirectMysqlQuery {
 			innerJoin("accessioncollect", "col", "col.accessionId=a.id");
 		}
 		
+		if (hasFilter(jsonTree, "storage")) {
+			innerJoin("accessionstorage", "storage", "storage.accessionId=a.id");
+		}
+		
 		return this;
 	}
 
@@ -128,6 +132,7 @@ public class DirectMysqlQuery {
 		createQuery(whereBuffer, "crop.shortName", jsonTree.get("crop"), params);
 		createQuery(whereBuffer, "accename.name", jsonTree.get("acceName"), params);
 		createQuery(whereBuffer, "col.collMissId", jsonTree.get("collMissId"), params);
+		createQuery(whereBuffer, "storage.storage", jsonTree.get("storage"), params);
 
 		for (final Iterator<String> it = jsonTree.fieldNames(); it.hasNext();) {
 			final String fieldName = it.next();

src/main/java/org/genesys2/server/service/impl/GenesysFilterServiceImpl.java

@@ -128,6 +128,8 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
 		this.availableFilters.add(new GenesysFilterImpl("inTrust", DataType.BOOLEAN));
 		this.availableFilters.add(new GenesysFilterImpl("available", DataType.BOOLEAN));
 		this.availableFilters.add(new GenesysFilterImpl("collMissId", DataType.STRING));
+		this.availableFilters.add(new GenesysI18nListFilterImpl<Integer>("storage", DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12,
+				13, 20, 30, 40, 50, 99 }));
 	}
 
 	@Override

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -236,12 +236,18 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 
 	@Override
 	public Accession getAccession(AccessionIdentifier3 aid3) {
-		return accessionRepository.findOne(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
+		Accession accession = accessionRepository.findOne(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
+		if (accession != null)
+			accession.getStoRage().size();
+		return accession;
 	}
 
 	@Override
 	public Accession getAccession(String instCode, String acceNumb) {
-		return accessionRepository.findByInstituteCodeAndAccessionName(instCode, acceNumb);
+		Accession accession = accessionRepository.findByInstituteCodeAndAccessionName(instCode, acceNumb);
+		if (accession != null)
+			accession.getStoRage().size();
+		return accession;
 	}
 
 	@Override
@@ -249,12 +255,18 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 		if (genus == null) {
 			throw new NullPointerException("Genus is required to load accession by instCode, acceNumb and genus");
 		}
-		return accessionRepository.findOne(instCode, acceNumb, genus);
+		Accession accession = accessionRepository.findOne(instCode, acceNumb, genus);
+		if (accession != null)
+			accession.getStoRage().size();
+		return accession;
 	}
 
 	@Override
 	public Accession getAccession(long accessionId) {
-		return accessionRepository.findOne(accessionId);
+		Accession accession = accessionRepository.findOne(accessionId);
+		if (accession != null)
+			accession.getStoRage().size();
+		return accession;
 	}
 
 	@Override
@@ -416,8 +428,8 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 	}
 
 	@Override
-	public Page<Accession> listAccessions(Pageable pageable) {
-		return accessionRepository.findAll(pageable);
+	public List<Long> listAccessionsIds(Pageable pageable) {
+		return accessionRepository.listAccessionsIds(pageable);
 	}
 
 	@Override
@@ -540,7 +552,8 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 	@Transactional(readOnly = false)
 	public void saveAccession(Accession... accession) {
 		for (final Accession a : accession) {
-			LOG.info("Updating " + a);
+			if (LOG.isDebugEnabled())
+				LOG.debug("Updating " + a);
 			accessionRepository.save(a);
 		}
 	}
@@ -601,13 +614,13 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 	public void saveExchange(List<AccessionExchange> all) {
 		accessionExchangeRepository.save(all);
 	}
-	
+
 	@Override
 	@Transactional(readOnly = false)
 	public void saveRemarks(List<AccessionRemark> toSaveRemarks) {
 		accessionRemarkRepository.save(toSaveRemarks);
 	}
-	
+
 	@Override
 	@Transactional(readOnly = false)
 	public void removeRemarks(List<AccessionRemark> toRemoveRemarks) {

src/main/java/org/genesys2/server/service/worker/AccessionStorageScanner.java

+/**
+ * Copyright 2014 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.genesys2.server.service.worker;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.genesys2.server.aspect.AsAdmin;
+import org.genesys2.server.model.genesys.Accession;
+import org.genesys2.server.service.GenesysService;
+import org.genesys2.server.service.impl.BatchRESTServiceImpl;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.stereotype.Component;
+
+/**
+ * Scans Accession#storage records and updates Accession#stoRage
+ */
+@Component
+public class AccessionStorageScanner {
+	public static final Log LOG = LogFactory.getLog(AccessionStorageScanner.class);
+
+	@Autowired
+	private GenesysService genesysService;
+
+	private static final int BATCH_SIZE = 50;
+
+	@AsAdmin
+	public void scanStorage() throws InterruptedException {
+		int page = 0;
+
+		do {
+			page++;
+			if (page % 100 == 0) {
+				LOG.info("Scanning accessions page " + page);
+			} else {
+				LOG.debug("Scanning accessions page " + page);
+			}
+			List<Long> accns = genesysService.listAccessionsIds(new PageRequest(page, BATCH_SIZE));
+			List<Accession> toSave = new ArrayList<Accession>(accns.size());
+
+			if (accns.size() == 0)
+				break;
+
+			for (long accnId : accns) {
+				Accession accession = genesysService.getAccession(accnId);
+				boolean updated = BatchRESTServiceImpl.updateAccessionStorage(accession, accession.getStorage());
+				if (updated) {
+					toSave.add(accession);
+				}
+			}
+
+			try {
+				if (toSave.size() > 0) {
+					LOG.info("Updating accession#stoRage for size=" + toSave.size());
+					genesysService.saveAccession(toSave.toArray(new Accession[] {}));
+				}
+			} catch (Throwable e) {
+				LOG.warn(e.getMessage(), e);
+			}
+
+			Thread.sleep(5);
+		} while (true);
+
+		LOG.info("Done scanning accession#storage");
+	}
+}

src/main/java/org/genesys2/server/servlet/controller/AdminController.java

@@ -36,12 +36,14 @@ import org.genesys2.server.service.InstituteService;
 import org.genesys2.server.service.LuceneIndexer;
 import org.genesys2.server.service.MappingService;
 import org.genesys2.server.service.impl.ContentSanitizer;
+import org.genesys2.server.service.worker.AccessionStorageScanner;
 import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
 import org.genesys2.server.service.worker.InstituteUpdater;
 import org.genesys2.server.service.worker.SGSVUpdate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.cache.Cache;
 import org.springframework.cache.CacheManager;
+import org.springframework.core.task.TaskExecutor;
 import org.springframework.jdbc.core.RowCallbackHandler;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.Model;
@@ -92,6 +94,12 @@ public class AdminController {
 	@Autowired
 	private CacheManager cacheManager;
 
+	@Autowired
+	private TaskExecutor taskExecutor;
+
+	@Autowired
+	protected AccessionStorageScanner accessionStorageScanner;
+
 	@RequestMapping("/")
 	public String root(Model model) {
 		return "/admin/index";
@@ -308,14 +316,40 @@ public class AdminController {
 		return "redirect:/admin/";
 	}
 
+	@RequestMapping(method = RequestMethod.POST, value = "/scanStorage")
+	public String scanStorage() {
+		LOG.info("Updating accession.stoRage");
+		taskExecutor.execute(new Runnable() {
+			@Override
+			public void run() {
+				Thread t = Thread.currentThread();
+				int prevPrio = t.getPriority();
+				String prevName = t.getName();
+				t.setPriority(Thread.MIN_PRIORITY);
+				t.setName("background.storageScanner");
+				try {
+					accessionStorageScanner.scanStorage();
+					LOG.info("Updating done");
+				} catch (InterruptedException e) {
+					LOG.warn("Storage scanner thread interrupted!");
+				} finally {
+					LOG.debug("Resetting thread priority");
+					t.setPriority(prevPrio);
+					t.setName(prevName);
+				}
+			}
+		});
+		LOG.info("Job scheduled");
+		return "redirect:/admin/";
+	}
+
 	@RequestMapping(method = RequestMethod.POST, value = "/clearTilesCache")
 	public String clearTilesCache() {
 		final Cache tileServerCache = cacheManager.getCache("tileserver");
 		System.err.println("tileServerCache=" + tileServerCache.getNativeCache());
 
 		@SuppressWarnings("rawtypes")
-		final
-		IMap hazelCache = (IMap) tileServerCache.getNativeCache();
+		final IMap hazelCache = (IMap) tileServerCache.getNativeCache();
 
 		LOG.info("Tiles cache size=" + hazelCache.size());
 		int count = 0;

src/main/resources/content/language.properties

@@ -350,6 +350,8 @@ filter.close=Close
 filter.remove=Remove filter
 filter.autocomplete-placeholder=Type more than 3 characters...
 filter.collMissId=Collecting mission ID 
+filter.storage=Type of Germplasm storage
+
 
 search.page.title=Full-text Search
 search.no-results=No matches found for your query.

src/main/webapp/WEB-INF/jsp/accession/details.jsp

@@ -133,9 +133,16 @@
 
 			<div class="row">
 				<div class="col-xs-4"><spring:message code="accession.storage" /></div>
-				<div class="col-xs-8"><c:forEach items="${accession.storage.split('[;,]')}" var="storage">
-					<div><spring:message code="accession.storage.${storage}" /></div>
-				</c:forEach></div>
+				<div class="col-xs-8">
+					<c:forEach items="${accession.storage.split('[;,]')}" var="storage">
+						<div><spring:message code="accession.storage.${storage}" /></div>
+					</c:forEach>
+<%--				
+					<c:forEach items="${accession.stoRage}" var="storage">
+						<div><spring:message code="accession.storage.${storage}" /></div>
+					</c:forEach>
+--%>
+				</div>
 			</div>
 
 			<div class="row">

src/main/webapp/WEB-INF/jsp/admin/index.jsp

@@ -73,7 +73,11 @@
         <!-- CSRF protection -->
         <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
 	</form>
-	
+	<form method="post" action="<c:url value="/admin/scanStorage" />">
+		<input type="submit" class="btn btn-default" class="btn btn-default" value="Scan and convert STORAGE" />
+        <!-- CSRF protection -->
+        <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
+	</form>
 	<h3>C&E</h3>
 	<form method="post" action="<c:url value="/admin/refreshMetadataMethods" />">
 		<input type="submit" class="btn btn-default" class="btn btn-default" value="Recalculate metadata methods" />