Commit 25b6be95 authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch 'master' into staging

* master:
  MCPDUtil#toStrings fixed and renamed to #toList
  Always update lastModifiedDate with REST upsert operation
  ElasticUpdaterProcessor uses @Scheduled annotation instead of own background thread
  opencsv upgraded
parents 35719f47 2cb88c13
......@@ -113,7 +113,7 @@ public class AccessionDetails {
ad.available = accession.getAvailability();
ad.historic = accession.getHistoric();
ad.orgCty = Country.from(accession.getCountryOfOrigin());
ad.duplSite = MCPDUtil.toStrings(accession.getDuplSite());
ad.duplSite = MCPDUtil.toList(accession.getDuplSite());
ad.institute = Institute.from(accession.getInstitute());
ad.mlsStatus = accession.getMlsStatus();
ad.art15 = accession.getInTrust();
......
......@@ -31,7 +31,7 @@ public class Collect {
Collect c = new Collect();
if (StringUtils.isNotBlank(collect.getCollCode()))
c.collCode = new HashSet<String>(MCPDUtil.toStrings(collect.getCollCode()));
c.collCode = new HashSet<String>(MCPDUtil.toList(collect.getCollCode()));
c.collDate = StringUtils.defaultIfBlank(collect.getCollDate(), null);
c.collInstAddr = StringUtils.defaultIfBlank(collect.getCollInstAddress(), null);
c.collMissId = StringUtils.defaultIfBlank(collect.getCollMissId(), null);
......
......@@ -19,6 +19,7 @@ package org.genesys2.server.service.impl;
import static org.genesys2.util.NumberUtils.areEqual;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
......@@ -44,6 +45,8 @@ import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.json.Api1Constants;
import org.genesys2.server.persistence.domain.AccessionCustomRepository;
import org.genesys2.server.service.AccessionOpResponse;
import org.genesys2.server.service.AccessionOpResponse.UpsertResult;
import org.genesys2.server.service.BatchRESTService;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.GenesysService;
......@@ -51,8 +54,6 @@ import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.AccessionOpResponse;
import org.genesys2.server.service.AccessionOpResponse.UpsertResult;
import org.genesys2.server.servlet.controller.rest.model.AccessionAliasJson;
import org.genesys2.server.servlet.controller.rest.model.AccessionHeaderJson;
import org.genesys2.server.servlet.controller.rest.model.AccessionNamesJson;
......@@ -149,6 +150,9 @@ public class BatchRESTServiceImpl implements BatchRESTService {
public List<AccessionOpResponse> upsertAccessionData(FaoInstitute institute, Map<AccessionHeaderJson, ObjectNode> batch) throws RESTApiException {
LOG.info("Batch processing " + batch.size() + " entries for " + institute.getCode());
// Modify date
Date modifyDate = new Date();
final boolean useUniqueAcceNumbs = institute.hasUniqueAcceNumbs();
final List<AccessionOpResponse> upsertResponses = new ArrayList<AccessionOpResponse>();
......@@ -199,12 +203,10 @@ public class BatchRESTServiceImpl implements BatchRESTService {
throw new RESTApiException("Accession does not belong to instCode=" + institute.getCode() + " acn=" + dataJson);
}
Accession accession = loaded
.stream()
.filter(a -> useUniqueAcceNumbs ? (a.getAccessionName().equalsIgnoreCase(dataJson.acceNumb)) : (a.getAccessionName().equalsIgnoreCase(
dataJson.acceNumb) && a.getTaxonomy().getGenus().equalsIgnoreCase(dataJson.genus))).findFirst().orElse(null);
boolean updated = false;
Accession accession = loaded.stream()
.filter(a -> useUniqueAcceNumbs ? (a.getAccessionName().equalsIgnoreCase(dataJson.acceNumb))
: (a.getAccessionName().equalsIgnoreCase(dataJson.acceNumb) && a.getTaxonomy().getGenus().equalsIgnoreCase(dataJson.genus)))
.findFirst().orElse(null);
final ObjectNode accnJson = batch.get(dataJson);
......@@ -222,7 +224,6 @@ public class BatchRESTServiceImpl implements BatchRESTService {
throw new RESTApiException("Cannot create new accession without specifying genus");
}
updated = true;
upsertResult = new UpsertResult(UpsertResult.Type.INSERT);
} else {
......@@ -238,25 +239,22 @@ public class BatchRESTServiceImpl implements BatchRESTService {
|| accnJson.get(Api1Constants.Accession.SPAUTHOR) != null || accnJson.get(Api1Constants.Accession.SUBTAXA) != null
|| accnJson.get(Api1Constants.Accession.SUBTAUTHOR) != null) {
updated |= updateTaxonomy(accession, accnJson);
updateTaxonomy(accession, accnJson);
}
updated |= updateCrop(accession, accnJson.get(Api1Constants.Accession.CROPNAME));
updated |= updateAcceNumb(accession, accnJson.get(Api1Constants.Accession.ACCENUMB_NEW));
updated |= updateOrgCty(accession, accnJson.get(Api1Constants.Accession.ORIGCTY));
updated |= updateUuid(accession, accnJson.get(Api1Constants.Accession.UUID));
updated |= updateRemarks(accession, accnJson.get(Api1Constants.Accession.REMARKS), toSaveRemarks, toRemoveRemarks);
updated |= updateStorage(accession, accnJson);
updateCrop(accession, accnJson.get(Api1Constants.Accession.CROPNAME));
updateAcceNumb(accession, accnJson.get(Api1Constants.Accession.ACCENUMB_NEW));
updateOrgCty(accession, accnJson.get(Api1Constants.Accession.ORIGCTY));
updateUuid(accession, accnJson.get(Api1Constants.Accession.UUID));
updateRemarks(accession, accnJson.get(Api1Constants.Accession.REMARKS), toSaveRemarks, toRemoveRemarks);
updateStorage(accession, accnJson);
// TODO Move other setters to methods
JsonNode value = accnJson.get(Api1Constants.Accession.ACQDATE);
if (value != null) {
final String acqDate = value.isNull() ? null : value.textValue();
if (!StringUtils.equals(acqDate, accession.getAcquisitionDate())) {
accession.setAcquisitionDate(acqDate);
updated = true;
}
accession.setAcquisitionDate(acqDate);
}
value = accnJson.get(Api1Constants.Accession.MLSSTAT);
if (value != null) {
......@@ -264,11 +262,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
throw new RESTApiDataTypeException("If provided, 'mlsStat' must be a boolean");
}
final Boolean inMls = value.isNull() ? null : value.asBoolean();
if (!areEqual(inMls, accession.getMlsStatus())) {
accession.setMlsStatus(inMls);
updated = true;
}
accession.setMlsStatus(inMls);
}
value = accnJson.get(Api1Constants.Accession.INTRUST);
if (value != null) {
......@@ -276,11 +270,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
throw new RESTApiDataTypeException("If provided, 'inTrust' must be a boolean");
}
final Boolean inTrust = value.isNull() ? null : value.asBoolean();
if (!areEqual(inTrust, accession.getInTrust())) {
accession.setInTrust(inTrust);
updated = true;
}
accession.setInTrust(inTrust);
}
value = accnJson.get(Api1Constants.Accession.AVAILABLE);
if (value != null) {
......@@ -288,11 +278,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
throw new RESTApiDataTypeException("If provided, 'available' must be a boolean");
}
final Boolean availability = value.isNull() ? null : value.asBoolean();
if (!areEqual(availability, accession.getAvailability())) {
accession.setAvailability(availability);
updated = true;
}
accession.setAvailability(availability);
}
value = accnJson.get(Api1Constants.Accession.HISTORIC);
if (value != null) {
......@@ -300,10 +286,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
throw new RESTApiDataTypeException("If provided, 'historic' must be a boolean");
}
final boolean historic = value.asBoolean();
if (!areEqual(historic, accession.getHistoric())) {
accession.setHistoric(historic);
updated = true;
}
accession.setHistoric(historic);
}
value = accnJson.get(Api1Constants.Accession.ACCENAME);
......@@ -322,19 +305,13 @@ public class BatchRESTServiceImpl implements BatchRESTService {
throw new RESTApiDataTypeException("If provided, 'sampStat' must be a number");
}
final Integer sampStat = value.isNull() || !value.isNumber() ? null : value.asInt();
if (!areEqual(sampStat, accession.getSampleStatus())) {
accession.setSampleStatus(sampStat);
updated = true;
}
accession.setSampleStatus(sampStat);
}
value = accnJson.get(Api1Constants.Accession.DUPLSITE);
if (value != null) {
final String duplSite = arrayToString(toMcpdArray(accnJson, Api1Constants.Accession.DUPLSITE));
if (!StringUtils.equals(duplSite, accession.getDuplSite())) {
accession.setDuplSite(StringUtils.defaultIfBlank(duplSite, null));
updated = true;
}
accession.setDuplSite(StringUtils.defaultIfBlank(duplSite, null));
}
value = accnJson.get(Api1Constants.Accession.ACCEURL);
......@@ -343,11 +320,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
throw new RESTApiDataTypeException("If provided, 'acceUrl' must be a String");
}
final String acceUrl = value.isNull() ? null : value.textValue();
if (!areEqual(acceUrl, accession.getAcceUrl())) {
accession.setAcceUrl(acceUrl);
updated = true;
}
accession.setAcceUrl(acceUrl);
}
if (accnJson.has(Api1Constants.Accession.COLL)) {
......@@ -512,10 +485,8 @@ public class BatchRESTServiceImpl implements BatchRESTService {
}
}
if (updated) {
toSave.add(accession);
}
accession.setLastModifiedDate(modifyDate);
toSave.add(accession);
// Set upsert result
upsertResponse.setResult(upsertResult);
}
......@@ -525,24 +496,21 @@ public class BatchRESTServiceImpl implements BatchRESTService {
List<Accession> savedData = genesysService.saveAccessions(institute, toSave);
// Iterate savedData to extract UUIDs
upsertResponses.stream().forEach(
response -> {
Accession accession = savedData
.stream()
.filter(a -> useUniqueAcceNumbs ? (a.getAccessionName().equalsIgnoreCase(response.getAcceNumb())) : (a.getAccessionName()
.equalsIgnoreCase(response.getAcceNumb()) && a.getTaxonomy().getGenus().equalsIgnoreCase(response.getGenus())))
.findFirst().orElse(null);
UpsertResult result = response.getResult();
if (accession != null) {
if (result.getUuid() == null) {
result.setAction(UpsertResult.Type.INSERT);
} else {
result.setAction(UpsertResult.Type.UPDATE);
}
result.setUUID(accession.getUuid());
}
});
upsertResponses.stream().forEach(response -> {
Accession accession = savedData.stream().filter(a -> useUniqueAcceNumbs ? (a.getAccessionName().equalsIgnoreCase(response.getAcceNumb()))
: (a.getAccessionName().equalsIgnoreCase(response.getAcceNumb()) && a.getTaxonomy().getGenus().equalsIgnoreCase(response.getGenus())))
.findFirst().orElse(null);
UpsertResult result = response.getResult();
if (accession != null) {
if (result.getUuid() == null) {
result.setAction(UpsertResult.Type.INSERT);
} else {
result.setAction(UpsertResult.Type.UPDATE);
}
result.setUUID(accession.getUuid());
}
});
}
if (toSaveColl.size() > 0) {
......@@ -799,8 +767,8 @@ public class BatchRESTServiceImpl implements BatchRESTService {
*/
private boolean updateTaxonomy(AccessionData accession, JsonNode accnJson) throws RESTApiException {
if (accession.getTaxonomy() != null && (!accnJson.has(Api1Constants.Accession.GENUS_NEW) && !accnJson.has(Api1Constants.Accession.SPECIES))) {
if (LOG.isWarnEnabled())
LOG.warn("Not updating taxonomy without newGenus or species data");
if (LOG.isDebugEnabled())
LOG.debug("Not updating taxonomy without newGenus or species data");
return false;
}
......
......@@ -16,50 +16,61 @@
package org.genesys2.util;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
public class MCPDUtil {
static Pattern mcpdSplit = Pattern.compile("\\w*;\\w*");
static Pattern mcpdSplit = Pattern.compile("\\s*;\\s*");
public static String toMcpdArray(List<Integer> integers) {
if (integers == null || integers.size() == 0)
/**
* Convert a List to MCPD "array string" with semicolon as separator.
*
* @param integers
* @return
*/
public static String toMcpdArray(List<?> objects) {
if (objects == null || objects.size() == 0)
return null;
String s = "";
for (Integer i : integers) {
if (i == null)
continue;
if (s.length() > 0)
s += ";";
s += i;
}
return StringUtils.defaultIfBlank(s, null);
StringBuilder sb = new StringBuilder();
objects.stream().filter(i -> i != null).forEach(i -> {
if (sb.length() > 0)
sb.append(";");
sb.append(i);
});
return StringUtils.defaultIfBlank(sb.toString(), null);
}
/**
* Convert an MCPD "array" (e.g. "10;20") to a {@link List} of Integers,
* excluding blank and null elements.
*
* @throws NumberFormatException
* if the input string contains non-numeric elements.
*/
public static List<Integer> toIntegers(String ints) {
if (StringUtils.isBlank(ints))
return null;
List<Integer> is = new ArrayList<Integer>();
for (String i : mcpdSplit.split(ints)) {
if (StringUtils.isBlank(i))
continue;
try {
is.add(Integer.parseInt(i));
} catch (NumberFormatException e) {
}
}
return is;
return Arrays.stream(mcpdSplit.split(ints)).filter(str -> StringUtils.isNotBlank(str)).map(i -> Integer.parseInt(i)).collect(Collectors.toList());
}
public static List<String> toStrings(String strings) {
/**
* Convert an MCPD "array string" (e.g. "NOR051;AUS033") to a {@link List},
* excluding blank and null elements.
*
* @param strings
* @return
*/
public static List<String> toList(String strings) {
if (StringUtils.isBlank(strings))
return null;
return Arrays.asList(mcpdSplit.split(strings));
return Arrays.stream(mcpdSplit.split(strings)).filter(str -> StringUtils.isNotBlank(str)).collect(Collectors.toList());
}
}
/**
* Copyright 2016 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.model.impl;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertThat;
import java.util.ArrayList;
import java.util.List;
import org.genesys2.util.MCPDUtil;
import org.junit.Test;
public class MCPDUtilTest {
@Test
public void testMcpdArray() {
assertThat("Result should be null", MCPDUtil.toMcpdArray(null), is(nullValue()));
List<Integer> ints = new ArrayList<Integer>();
assertThat("Result should be null", MCPDUtil.toMcpdArray(ints), is(nullValue()));
ints.add(1);
assertThat("Result incorrect", MCPDUtil.toMcpdArray(ints), equalTo("1"));
ints.add(2);
assertThat("Result incorrect", MCPDUtil.toMcpdArray(ints), equalTo("1;2"));
ints.add(3);
assertThat("Result incorrect", MCPDUtil.toMcpdArray(ints), equalTo("1;2;3"));
ints.remove(0);
assertThat("Result incorrect", MCPDUtil.toMcpdArray(ints), equalTo("2;3"));
ints.add(null);
assertThat("Result incorrect", MCPDUtil.toMcpdArray(ints), equalTo("2;3"));
ints.add(4);
assertThat("Result incorrect", MCPDUtil.toMcpdArray(ints), equalTo("2;3;4"));
}
@Test
public void testToStrings() {
assertThat("List should be null", MCPDUtil.toList(null), is(nullValue()));
assertThat("List should have 1 element", MCPDUtil.toList("NOR051;"), hasSize(equalTo(1)));
assertThat("Missing element", MCPDUtil.toList("NOR051;"), contains("NOR051"));
assertThat("List should have 1 element", MCPDUtil.toList(";NOR051;"), hasSize(equalTo(1)));
assertThat("Missing element", MCPDUtil.toList(";NOR051;"), contains("NOR051"));
assertThat("List should have 1 element", MCPDUtil.toList("NOR051;;"), hasSize(equalTo(1)));
assertThat("Missing element", MCPDUtil.toList("NOR051;;"), contains("NOR051"));
assertThat("List should have 2 elements", MCPDUtil.toList("AUS053;NOR051"), hasSize(equalTo(2)));
assertThat("Missing element", MCPDUtil.toList("AUS053;NOR051"), contains("AUS053", "NOR051"));
assertThat("List should have 2 elements", MCPDUtil.toList("; ; AUS053;NOR051"), hasSize(equalTo(2)));
assertThat("Missing element", MCPDUtil.toList("; ; AUS053;NOR051"), contains("AUS053", "NOR051"));
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment