Commit a082e049 authored by Matija Obreza's avatar Matija Obreza

Accession API v1: Uploader

- Unit test
parent 70067418
......@@ -79,14 +79,14 @@ import io.swagger.annotations.Api;
*/
@RestController("accessionApi1")
@PreAuthorize("isAuthenticated()")
@RequestMapping(AccessionController.API_BASE)
@RequestMapping(AccessionController.CONTROLLER_URL)
@Api(tags = { "accession" })
public class AccessionController {
private static final Logger LOG = LoggerFactory.getLogger(AccessionController.class);
/** The Constant API_BASE. */
public static final String API_BASE = ApiBaseController.APIv1_BASE + "/acn";
/** The Constant CONTROLLER_URL. */
public static final String CONTROLLER_URL = ApiBaseController.APIv1_BASE + "/acn";
private static final int DOWNLOAD_LIMIT = 200000;
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.api.v1;
import java.util.ArrayList;
import java.util.List;
import javax.persistence.PersistenceException;
import javax.persistence.RollbackException;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.genesys2.server.api.ApiBaseController;
import org.genesys2.server.api.PleaseRetryException;
import org.genesys2.server.api.model.AccessionHeaderJson;
import org.genesys2.server.exception.InvalidApiUsageException;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.impl.RESTApiException;
import org.genesys2.server.service.worker.AccessionOpResponse;
import org.genesys2.server.service.worker.AccessionOpResponse.UpsertResult;
import org.genesys2.server.service.worker.AccessionUploader;
import org.genesys2.spring.ResourceNotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.CannotAcquireLockException;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.transaction.TransactionException;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import io.swagger.annotations.Api;
/**
* Accession API v1: the uploader.
*/
@RestController("accessionUploadApi1")
@PreAuthorize("isAuthenticated()")
@RequestMapping(AccessionUploadController.CONTROLLER_URL)
@Api(tags = { "accession" })
public class AccessionUploadController {
/** The Constant LOG. */
private static final Logger LOG = LoggerFactory.getLogger(AccessionUploadController.class);
/** The Constant CONTROLLER_URL. */
public static final String CONTROLLER_URL = ApiBaseController.APIv1_BASE + "/acn";
/** The Constant UPLOAD_RETRIES. */
private static final int UPLOAD_RETRIES = 10;
/** The uploader. */
@Autowired
private AccessionUploader uploader;
/** The institute service. */
@Autowired
InstituteService instituteService;
/**
* Update accessions in the system.
*
* @param instCode the WIEWS institute code
* @param updates the updates
* @return the list
* @throws Exception the exception
*/
@PreAuthorize("isAuthenticated()")
@RequestMapping(value = "/{instCode}/upsert", method = { RequestMethod.POST }, produces = { MediaType.APPLICATION_JSON_VALUE })
public List<AccessionOpResponse> upsertInstituteAccession(@PathVariable("instCode") String instCode, @RequestBody ArrayNode updates) throws Exception {
StopWatch stopWatch = StopWatch.createStarted();
LOG.trace("Received:\n {}", updates);
// User's permission to WRITE to this WIEWS institute are checked in
// BatchRESTService.
final FaoInstitute institute = instituteService.getInstitute(instCode);
if (institute == null) {
throw new ResourceNotFoundException();
}
for (int tryCount = 0; tryCount < UPLOAD_RETRIES; tryCount++) {
try {
List<AccessionOpResponse> res = uploader.upsertAccessions(institute, updates);
LOG.info("Processed {} accessions for {} in {} tries in {}ms", updates.size(), instCode, tryCount + 1, stopWatch.getTime());
return res;
} catch (DataAccessException | TransactionException | PersistenceException e) {
List<AccessionOpResponse> res = upsert1By1(institute, updates);
LOG.info("Processed {} accessions for {} 1by1 after {} tries in {}ms because of {}", updates.size(), instCode, tryCount + 1, stopWatch.getTime(), e.getMessage());
return res;
} catch (PleaseRetryException e) {
LOG.error("Retry {} of {} tries due to: {}", tryCount, UPLOAD_RETRIES, e.getMessage());
// Wait a bit
Thread.sleep((long) (RandomUtils.nextDouble(10, 100) * tryCount));
}
}
throw new InvalidApiUsageException("Just couldn't do it.");
}
/**
* Upsert 1 by 1.
*
* @param institute the institute
* @param updates the updates
* @return the list
* @throws Exception the exception
*/
private List<AccessionOpResponse> upsert1By1(FaoInstitute institute, ArrayNode updates) throws Exception {
List<AccessionOpResponse> response = new ArrayList<>();
ArrayNode single = new ObjectMapper().createArrayNode();
for (JsonNode update : updates) {
single.removeAll();
single.add(update);
Throwable lastException = null;
for (int tryCount = 0; tryCount < UPLOAD_RETRIES; tryCount++) {
try {
response.addAll(uploader.upsertAccessions(institute, single));
lastException = null;
break;
} catch (TransactionException | RollbackException | CannotAcquireLockException | DataIntegrityViolationException | PleaseRetryException e) {
lastException = e;
if (tryCount > 1) {
LOG.info("Retry {} of {} tries due to: {}", tryCount, UPLOAD_RETRIES, e.getMessage());
} else {
LOG.debug("Retrying {} of {} tries due to: {}", tryCount, UPLOAD_RETRIES, e.getMessage());
}
// Wait a bit
Thread.sleep((long) (RandomUtils.nextDouble(10, 100) * tryCount));
}
}
if (lastException != null) {
LOG.warn("Upsert failed due to: {} data={}", lastException.getMessage(), update, lastException);
// record error
response.add(new AccessionOpResponse(update.get("instituteCode").asText(), update.get("accessionNumber").asText(), update.get("taxonomy").get("genus").asText())
.setResult(new UpsertResult(UpsertResult.Type.ERROR)).setError(getDetailedErrorMessage(lastException)));
}
}
return response;
}
/**
* Delete accessions by instituteCode, acceNumb and genus.
*
* @param instCode the inst code
* @param batch the batch
* @return the list
* @throws RESTApiException the REST api exception
*/
@PreAuthorize("isAuthenticated()")
@RequestMapping(value = "/{instCode}/delete", method = { RequestMethod.POST }, consumes = { MediaType.APPLICATION_JSON_VALUE }, produces = {
MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody List<AccessionOpResponse> deleteAccessions(@PathVariable("instCode") String instCode, @RequestBody List<AccessionHeaderJson> batch) throws RESTApiException {
// User's permission to WRITE to this WIEWS institute are checked in
// BatchRESTService.
final FaoInstitute institute = instituteService.getInstitute(instCode);
if (institute == null) {
throw new ResourceNotFoundException();
}
try {
List<AccessionOpResponse> response = null;
try {
response = uploader.deleteAccessions(institute, batch);
LOG.info("Deleted {} accessions from {}", response.size(), instCode);
} catch (DataAccessException | TransactionException | PersistenceException e) {
LOG.info("Retrying delete one by one due to {}", e.getMessage());
response = deleteAccessions1by1(institute, batch);
}
return response;
} catch (CannotAcquireLockException e) {
throw new PleaseRetryException("Operation failed, please retry.", e);
}
}
/**
* Delete accessions 1 by 1.
*
* @param institute the institute
* @param batch the batch
* @return the list
*/
private List<AccessionOpResponse> deleteAccessions1by1(FaoInstitute institute, List<AccessionHeaderJson> batch) {
LOG.info("Attempting delete 1 by 1");
final List<AccessionHeaderJson> batchOfOne = new ArrayList<AccessionHeaderJson>();
List<AccessionOpResponse> response = new ArrayList<AccessionOpResponse>();
for (AccessionHeaderJson acceJ : batch) {
try {
batchOfOne.clear();
batchOfOne.add(acceJ);
AccessionOpResponse accessionResponse = uploader.deleteAccessions(institute, batchOfOne).get(0);
response.add(accessionResponse);
} catch (Throwable e) {
if (LOG.isInfoEnabled()) {
LOG.info("Error deleting {}: {}", acceJ.instCode, e.getMessage());
}
AccessionOpResponse accessionResponse = new AccessionOpResponse(acceJ.instCode, acceJ.acceNumb, acceJ.genus);
accessionResponse.setError(getDetailedErrorMessage(e));
response.add(accessionResponse);
}
}
return response;
}
/**
* Gets the detailed error message.
*
* @param e the exception
* @return the detailed error message
*/
private String getDetailedErrorMessage(Throwable e) {
StringBuffer sb=new StringBuffer(e.getMessage());
Throwable r = e;
while ((r = r.getCause()) !=null) {
sb.append("\n" + r.getMessage());
}
return sb.toString();
}
}
......@@ -140,6 +140,9 @@ public class AccessionUploader implements InitializingBean {
LOG.trace("Received: {}", accn);
try {
Accession accession = reader.readValue(accn);
if (accession.getInstituteCode() == null || (accession.getDoi() == null && accession.getAccessionNumber() == null)) {
throw new InvalidApiUsageException("instituteCode, accessionNumber OR doi missing in " + accn);
}
accessions.add(accession);
responses.add(new AccessionOpResponse(accession.getInstituteCode(), accession.getAccessionNumber(), accession.getTaxonomy().getGenus()));
} catch (IOException e) {
......
......@@ -29,7 +29,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.genesys.test.base.AbstractApiTest;
import org.genesys2.server.api.v0.AccessionController;
import org.genesys2.server.api.v1.AccessionController;
import org.genesys2.server.api.v1.AccessionUploadController;
import org.genesys2.server.model.genesys.*;
import org.genesys2.server.model.genesys.AccessionAlias.AliasType;
import org.genesys2.server.model.impl.FaoInstitute;
......@@ -135,7 +136,7 @@ public class AccessionControllerTest extends AbstractApiTest {
final String s = "[" + verboseMapper.writeValueAsString(accessionJson) + "]";
/*@formatter:off*/
mockMvc.perform(post(AccessionController.API_BASE + "/" + institute.getCode() + "/upsert")
mockMvc.perform(post(AccessionUploadController.CONTROLLER_URL + "/" + institute.getCode() + "/upsert")
.contentType(MediaType.APPLICATION_JSON)
.content(s))
// .andDo(MockMvcResultHandlers.print())
......@@ -168,7 +169,7 @@ public class AccessionControllerTest extends AbstractApiTest {
final String s = objectMapper.writeValueAsString(uuids);
/*@formatter:off*/
mockMvc.perform(post(org.genesys2.server.api.v1.AccessionController.API_BASE + "/for-uuid")
mockMvc.perform(post(AccessionController.CONTROLLER_URL + "/for-uuid")
.contentType(MediaType.APPLICATION_JSON)
.content(s))
.andExpect(status().isOk())
......@@ -286,13 +287,15 @@ public class AccessionControllerTest extends AbstractApiTest {
}
@Test
public void testUpgradeToArray() throws Exception {
public void testBreederCode() throws Exception {
ObjectNode accessionJson = setUpAccession();
upsert(accessionJson);
assertThat(accessionRepository.count(), is(1L));
accessionJson.put("breederCode", "BREEDER1");
ArrayNode arr = accessionJson.putArray("breederCode");
arr.add("BREEDER1");
arr.add("BREEDER2");
/*@formatter:off*/
upsert(accessionJson)
.andExpect(jsonPath("$[0].result.action", equalTo("UPDATE")));
......@@ -301,7 +304,7 @@ public class AccessionControllerTest extends AbstractApiTest {
assertThat(accessionRepository.count(), is(1L));
UUID uuid = accessionRepository.findAll().get(0).getUuid();
Accession result = fetch(uuid);
assertThat(result.getAccessionId().getBreederCode(), containsInAnyOrder("BREEDER1"));
assertThat(result.getAccessionId().getBreederCode(), containsInAnyOrder("BREEDER1", "BREEDER2"));
accessionJson.set("breederCode", null);
upsert(accessionJson);
......@@ -369,7 +372,7 @@ public class AccessionControllerTest extends AbstractApiTest {
private ResultActions upsert(ObjectNode accessionJson) throws Exception, JsonProcessingException {
LOG.debug("Upsering {}", verboseMapper.writerWithDefaultPrettyPrinter().writeValueAsString(accessionJson));
/*@formatter:off*/
return mockMvc.perform(post(AccessionController.API_BASE + "/" + institute.getCode() + "/upsert")
return mockMvc.perform(post(AccessionUploadController.CONTROLLER_URL + "/" + institute.getCode() + "/upsert")
.contentType(MediaType.APPLICATION_JSON)
.content("[" + verboseMapper.writeValueAsString(accessionJson) + "]"))
.andExpect(status().isOk())
......@@ -385,7 +388,7 @@ public class AccessionControllerTest extends AbstractApiTest {
private Accession fetch(UUID uuid) throws Exception, JsonProcessingException {
/*@formatter:off*/
String responseBody = mockMvc.perform(get(org.genesys2.server.api.v1.AccessionController.API_BASE+ "/" + uuid))
String responseBody = mockMvc.perform(get(org.genesys2.server.api.v1.AccessionController.CONTROLLER_URL+ "/" + uuid))
.andExpect(status().isOk())
// .andDo(MockMvcResultHandlers.print())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
......@@ -421,7 +424,7 @@ public class AccessionControllerTest extends AbstractApiTest {
private ObjectNode setUpAccession() {
ObjectNode accession = objectMapper.createObjectNode();
accession.put("instCode", institute.getCode());
accession.put("instituteCode", institute.getCode());
accession.put("accessionNumber", "A" + acceNumb.incrementAndGet());
ObjectNode taxa = accession.putObject("taxonomy");
taxa.put("genus", "Manihot");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment