Merge branch '26-sample-data-generation' into 'master'

Resolve "Sample data generation"

Closes #26

See merge request genesys-pgr/genesys-server!88

src/main/java/org/genesys2/server/aspect/AsAdminAspect.java

@@ -18,14 +18,14 @@ package org.genesys2.server.aspect;
 
 import java.util.Arrays;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.aspectj.lang.ProceedingJoinPoint;
 import org.aspectj.lang.annotation.Around;
 import org.aspectj.lang.annotation.Aspect;
 import org.genesys2.server.model.UserRole;
 import org.genesys2.server.model.impl.User;
 import org.genesys2.server.service.UserService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.security.core.Authentication;
 import org.springframework.security.core.authority.SimpleGrantedAuthority;
@@ -36,7 +36,7 @@ import org.springframework.stereotype.Component;
 @Aspect
 @Component
 public class AsAdminAspect {
-	private static final Log LOG = LogFactory.getLog(AsAdminAspect.class);
+	private static final Logger LOG = LoggerFactory.getLogger(AsAdminAspect.class);
 
 	@Autowired
 	private UserService userService;
@@ -71,7 +71,7 @@ public class AsAdminAspect {
 			return pjp.proceed();
 		} finally {
 			if (swapped) {
-				LOG.warn("Restoring privileges");
+				LOG.warn("Revoking ADMIN privileges");
 				SecurityContextHolder.getContext().setAuthentication(prevAuth);
 			}
 		}
@@ -86,7 +86,7 @@ public class AsAdminAspect {
 				LOG.warn("Temporary SYS_ADMIN account is being used.");
 				return new PreAuthenticatedAuthenticationToken("SYS_ADMIN", null, Arrays.asList(new SimpleGrantedAuthority(UserRole.ADMINISTRATOR.getAuthority())));
 			} else {
-				LOG.warn("Got SYS_ADMIN account: " + sysUser + " with roles=" + sysUser.getAuthorities());
+				LOG.warn("Got SYS_ADMIN account: {} with roles=", sysUser, sysUser.getAuthorities());
 				SYS_ADMIN = new PreAuthenticatedAuthenticationToken(sysUser, null, sysUser.getAuthorities());
 			}
 		}

src/main/java/org/genesys2/server/listener/RunAsAdminListener.java

@@ -18,9 +18,9 @@ package org.genesys2.server.listener;
 
 import java.util.concurrent.Callable;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.genesys2.server.security.AsAdminInvoker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -29,7 +29,7 @@ public abstract class RunAsAdminListener implements InitializingBean {
 	@Autowired
 	protected AsAdminInvoker asAdminInvoker;
 
-	protected Log _logger = LogFactory.getLog(getClass());
+	protected Logger LOG = LoggerFactory.getLogger(getClass());
 
 	// in initializing stage @AsAdmin doesn't work properly
 	@Override

src/main/java/org/genesys2/server/listener/sample/CreateAdminListener.java

@@ -37,7 +37,7 @@ public class CreateAdminListener extends RunAsAdminListener {
 
 	@Override
 	public void init() throws Exception {
-		_logger.info("Checking for at least one account");
+		LOG.info("Checking for at least one account");
 
 		if (userService.listUsers(new PageRequest(0, 1)).getTotalElements() == 0) {
 			createDefaultAccounts();
@@ -58,6 +58,6 @@ public class CreateAdminListener extends RunAsAdminListener {
 		
 		final User user = userService.createUser(email, fullName, password, accountType);
 		userService.setRoles(user, Sets.newHashSet(UserRole.ADMINISTRATOR));
-		_logger.warn("Admin account for " + email + " has been successfully added.");
+		LOG.warn("Admin account for " + email + " has been successfully added.");
 	}
 }

src/main/java/org/genesys2/server/listener/sample/CreateContentListener.java

@@ -60,7 +60,8 @@ import org.springframework.stereotype.Component;
 public class CreateContentListener extends RunAsAdminListener {
 
 	@Value("${auto.createContent}")
-	private boolean createContent = false;
+	// Must **not** be final!
+	private boolean createContent;
 
 	@Autowired
 	private ContentService contentService;
@@ -73,18 +74,18 @@ public class CreateContentListener extends RunAsAdminListener {
 
 	private void createArticles() throws IOException, JsonProcessingException {
 		if (!createContent) {
-			_logger.warn("Skipping content creation on startup.");
+			LOG.warn("Skipping content creation on startup.");
 			return;
 		}
 
-		_logger.debug("Checking if default content exists");
+		LOG.debug("Checking if default content exists");
 		final ClassLoader classLoader = CreateContentListener.class.getClassLoader();
 		final PathMatchingResourcePatternResolver rpr = new PathMatchingResourcePatternResolver(classLoader);
 		final String resourcePath = "/default-content/*";
 		final Resource[] rs = rpr.getResources(resourcePath);
 		for (final Resource r : rs) {
 			final String slug = r.getFilename().substring(0, r.getFilename().lastIndexOf(".json"));
-			_logger.info("Using " + r.getFilename() + " for article slug=" + slug);
+			LOG.info("Using " + r.getFilename() + " for article slug=" + slug);
 
 			final ObjectMapper mapper = new ObjectMapper();
 			try (InputStream stream = r.getInputStream()) {
@@ -102,14 +103,14 @@ public class CreateContentListener extends RunAsAdminListener {
 					if (article == null) {
 						try {
 							contentService.createGlobalArticle(slug, locale, entry.getValue().get("title").asText(), entry.getValue().get("body").asText(), null, false);
-							_logger.info("Created article for slug: " + slug + " lang=" + locale.getLanguage());
+							LOG.info("Created article for slug: " + slug + " lang=" + locale.getLanguage());
 						} catch (CRMException e) {
-							_logger.warn("Failed to create global article slug=" + slug + ".", e);
+							LOG.warn("Failed to create global article slug=" + slug + ".", e);
 						}
 					}
 				}
 			} catch (JsonParseException e) {
-				_logger.error("Could not create global article " + slug + " from " + r.getFilename() + ". JSON is not valid.");
+				LOG.error("Could not create global article " + slug + " from " + r.getFilename() + ". JSON is not valid.");
 			}
 		}
 	}
@@ -118,7 +119,7 @@ public class CreateContentListener extends RunAsAdminListener {
 	 * Generates CMS Menu and MenuItems
 	 */
 	private void createMenus() {
-		_logger.info("Checking if default menus exist");
+		LOG.info("Checking if default menus exist");
 
 		// legal
 		contentService.ensureMenuItem("legal", "/content/legal/disclaimer", "menu.disclaimer");

src/main/java/org/genesys2/server/listener/sample/FirstRunListener.java

-/**
- * Copyright 2014 Global Crop Diversity Trust
+/*
+ * Copyright 2017 Global Crop Diversity Trust
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,17 +12,38 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- **/
+ */
 
 package org.genesys2.server.listener.sample;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import org.apache.commons.lang.StringUtils;
+import org.genesys.blocks.oauth.service.OAuthClientDetailsService;
 import org.genesys2.server.listener.RunAsAdminListener;
+import org.genesys2.server.model.impl.FaoInstitute;
+import org.genesys2.server.persistence.domain.AccessionRepository;
+import org.genesys2.server.service.BatchRESTService;
+import org.genesys2.server.service.CropService;
 import org.genesys2.server.service.GeoRegionService;
 import org.genesys2.server.service.GeoService;
 import org.genesys2.server.service.InstituteService;
 import org.genesys2.server.service.worker.InstituteUpdater;
+import org.genesys2.server.servlet.controller.rest.model.AccessionHeaderJson;
+import org.genesys2.util.InvalidDOIException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.stereotype.Component;
 
@@ -35,7 +56,8 @@ import org.springframework.stereotype.Component;
 public class FirstRunListener extends RunAsAdminListener {
 
 	@Value("${auto.createContent}")
-	private final boolean createContent = false;
+	// Must **not** be final!
+	private boolean createContent;
 
 	@Autowired
 	GeoService geoService;
@@ -46,30 +68,112 @@ public class FirstRunListener extends RunAsAdminListener {
 	@Autowired
 	InstituteUpdater instituteUpdater;
 
-    @Autowired
-    GeoRegionService geoRegionService;
+	@Autowired
+	GeoRegionService geoRegionService;
+
+	@Autowired
+	CropService cropService;
+
+	@Autowired
+	OAuthClientDetailsService oauthClientService;
+
+	@Autowired
+	AccessionRepository accessionRepository;
+
+	@Autowired
+	BatchRESTService batchService;
 
 	@Override
 	protected void init() throws Exception {
 		if (!createContent) {
-			_logger.info("Skipping content creation on startup.");
+			LOG.warn("Skipping content creation on startup.");
 			return;
 		}
 
 		if (geoService.listAll().size() == 0) {
-			_logger.info("No country data found. Loading");
+			LOG.warn("No country data found. Loading");
 			geoService.updateCountryData();
 		}
 
 		if (instituteService.list(new PageRequest(0, 1)).getTotalElements() == 0) {
-			_logger.info("No WIEWS institutes found. Loading.");
+			LOG.warn("No WIEWS institutes found. Loading.");
 			instituteUpdater.updateFaoInstitutes();
 		}
-		
+
 		if (geoRegionService.findAll().isEmpty()) {
-            _logger.info("No geoRegion data found. Loading");
-            geoRegionService.updateGeoRegionData();
-        }
+			LOG.warn("No geoRegion data found. Loading");
+			geoRegionService.updateGeoRegionData();
+		}
+
+		if (cropService.listCrops().isEmpty()) {
+			LOG.warn("No crops data found. Adding");
+
+			addCrop("sweetpotato");
+			addCrop("sorghum");
+		}
+
+		if (oauthClientService.listClientDetails().isEmpty()) {
+			addDefaultOAuthClient();
+		} else {
+			LOG.warn("Skipping creation of initial OAuth client");
+		}
+
+		if (accessionRepository.count() == 0) {
+			addSomeAccessions();
+		} else {
+			LOG.warn("Skipping creation of dummy accession data");
+		}
+	}
+
+	private void addCrop(String shortName) {
+		LOG.info("No crops data found. Adding {}", shortName);
+		cropService.addCrop(shortName, StringUtils.capitalize(shortName), StringUtils.capitalize(shortName), null);
+	}
+
+	private void addDefaultOAuthClient() {
+		oauthClientService.addClient("Default Client", "System-generated initial OAuth client", "oob", null, null);
 	}
 
+	private void addSomeAccessions() throws IOException {
+		LOG.warn("Adding some passport data");
+
+		final ClassLoader classLoader = CreateContentListener.class.getClassLoader();
+		final PathMatchingResourcePatternResolver rpr = new PathMatchingResourcePatternResolver(classLoader);
+		final String resourcePath = "/dummy-data/*";
+		final Resource[] rs = rpr.getResources(resourcePath);
+		for (final Resource r : rs) {
+			final String instCode = r.getFilename().substring(0, r.getFilename().lastIndexOf(".json"));
+			LOG.info("Using " + r.getFilename() + " for instCode=" + instCode);
+
+			final ObjectMapper mapper = new ObjectMapper();
+			try (InputStream stream = r.getInputStream()) {
+				final JsonNode json = mapper.readTree(stream);
+
+				FaoInstitute institute = instituteService.getInstitute(instCode);
+				if (institute == null) {
+					throw new Exception("No Institute with instCode=" + instCode);
+				}
+
+				final Map<AccessionHeaderJson, ObjectNode> batch = new HashMap<>();
+				((ArrayNode) json).forEach(node -> {
+					try {
+						batch.put(AccessionHeaderJson.fromJson(node), (ObjectNode) node);
+					} catch (InvalidDOIException e) {
+						LOG.warn("Invalid DOI found in accession dummy data: " + r.getFilename(), e);
+					}
+				});
+
+				if (batch.size() > 0) {
+					LOG.warn("Upserting " + batch.size() + " accession records for " + instCode);
+					batchService.ensureTaxonomies(institute, batch);
+					batchService.upsertAccessionData(institute, batch);
+				}
+
+			} catch (JsonParseException e) {
+				LOG.error("Could not read dummy data for " + instCode + " from " + r.getFilename() + ". JSON is not valid.", e);
+			} catch (Exception e) {
+				LOG.error("Could not create dummy data. " + e.getMessage());
+			}
+		}
+	}
 }

src/main/java/org/genesys2/server/service/impl/GeoRegionServiceImpl.java

@@ -148,12 +148,13 @@ public class GeoRegionServiceImpl implements GeoRegionService {
 
 		List<GeoRegion> resultList = new ArrayList<>();
 
-		for (GeoRegion geoRegion: geoRegionList) {
-			if (geoRegion.getParentRegion().getName().equals("World") || geoRegion.getParentRegion().getName().equals("Americas")) {
+		for (GeoRegion geoRegion : geoRegionList) {
+			GeoRegion parentRegion = geoRegion.getParentRegion();
+			if (parentRegion != null && (parentRegion.getName().equals("World") || parentRegion.getName().equals("Americas"))) {
 				geoRegion.setParentRegion(null);
 				resultList.add(geoRegion);
-			}
-			else resultList.add(geoRegion);
+			} else
+				resultList.add(geoRegion);
 		}
 
 		Locale locale = LocaleContextHolder.getLocale();
@@ -168,9 +169,7 @@ public class GeoRegionServiceImpl implements GeoRegionService {
 		List<GeoRegion> subRegionsList = new ArrayList<>();
 		geoRegionList.removeIf(geoRegion -> geoRegion.getName().equals("World"));
 
-		subRegionsList.addAll(geoRegionList.stream().
-				filter(geoRegion -> parentGeo.getName().equals(geoRegion.getParentRegion().getName()))
-				.collect(Collectors.toList()));
+		subRegionsList.addAll(geoRegionList.stream().filter(geoRegion -> parentGeo.getName().equals(geoRegion.getParentRegion().getName())).collect(Collectors.toList()));
 
 		return subRegionsList;
 	}

src/main/java/org/genesys2/server/servlet/controller/UserControllerAdvice.java

@@ -86,7 +86,7 @@ public class UserControllerAdvice extends BaseController {
 	@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
 	@ExceptionHandler(value = { Throwable.class })
 	public ModelAndView handleAll(final HttpServletRequest req, final Throwable e) {
-        _logger.error("{} on {} {}", e.getMessage(), req.getMethod(), req.getRequestURL());
+		_logger.error("{} on {} {}", e.getMessage(), req.getMethod(), req.getRequestURL(), e);
 		final ModelAndView mav = new ModelAndView("/errors/error");
 		mav.addObject("exception", e);
 		return mav;

src/main/java/org/genesys2/server/servlet/controller/rest/AccessionController.java

@@ -34,7 +34,6 @@ import org.genesys2.server.model.elastic.AccessionDetails;
 import org.genesys2.server.model.genesys.Accession;
 import org.genesys2.server.model.impl.FaoInstitute;
 import org.genesys2.server.model.json.AccessionJson;
-import org.genesys2.server.model.json.Api1Constants;
 import org.genesys2.server.service.AccessionOpResponse;
 import org.genesys2.server.service.BatchRESTService;
 import org.genesys2.server.service.ElasticService;
@@ -52,7 +51,6 @@ import org.genesys2.server.service.impl.SearchException;
 import org.genesys2.server.servlet.controller.rest.model.AccessionHeaderJson;
 import org.genesys2.server.servlet.controller.rest.model.AccessionNamesJson;
 import org.genesys2.spring.ResourceNotFoundException;
-import org.genesys2.util.DOIUtils;
 import org.genesys2.util.InvalidDOIException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.dao.CannotAcquireLockException;
@@ -211,7 +209,7 @@ public class AccessionController extends RestController {
 		if (json.isArray()) {
 			for (final JsonNode j : json) {
 				try {
-					final AccessionHeaderJson dataJson = readAid3(j);
+					final AccessionHeaderJson dataJson = AccessionHeaderJson.fromJson(j);
 					if (!instCode.equals(dataJson.instCode)) {
 						throw new RuntimeException("Accession does not belong to instCode=" + instCode + " acn=" + dataJson);
 					}
@@ -222,7 +220,7 @@ public class AccessionController extends RestController {
 			}
 		} else {
 			try {
-				final AccessionHeaderJson dataJson = readAid3(json);
+				final AccessionHeaderJson dataJson = AccessionHeaderJson.fromJson(json);
 				if (!instCode.equals(dataJson.instCode)) {
 					throw new RuntimeException("Accession does not belong to instCode=" + instCode + " acn=" + dataJson);
 				}
@@ -496,20 +494,4 @@ public class AccessionController extends RestController {
 		public Integer maxRecords;
 		public String otherOptions;
 	}
-
-	private AccessionHeaderJson readAid3(JsonNode json) throws InvalidDOIException {
-		final AccessionHeaderJson dataJson = new AccessionHeaderJson();
-
-		dataJson.doi = json.has(Api1Constants.Accession.DOI) ? json.get(Api1Constants.Accession.DOI).textValue() : null;
-
-		if (dataJson.doi != null) {
-			DOIUtils.validateDoi(dataJson.doi);
-		}
-
-		dataJson.instCode = json.has(Api1Constants.Accession.INSTCODE) ? json.get(Api1Constants.Accession.INSTCODE).textValue() : null;
-		dataJson.acceNumb = json.has(Api1Constants.Accession.ACCENUMB) ? json.get(Api1Constants.Accession.ACCENUMB).textValue() : null;
-		dataJson.genus = json.has(Api1Constants.Accession.GENUS) ? json.get(Api1Constants.Accession.GENUS).textValue() : null;
-
-		return dataJson;
-	}
 }

src/main/java/org/genesys2/server/servlet/controller/rest/model/AccessionHeaderJson.java

@@ -18,7 +18,12 @@ package org.genesys2.server.servlet.controller.rest.model;
 
 import java.text.MessageFormat;
 
+import com.fasterxml.jackson.databind.JsonNode;
+
 import org.genesys2.server.model.impl.AccessionIdentifier3;
+import org.genesys2.server.model.json.Api1Constants;
+import org.genesys2.util.DOIUtils;
+import org.genesys2.util.InvalidDOIException;
 
 public class AccessionHeaderJson implements AccessionIdentifier3 {
 	public String doi;
@@ -26,11 +31,27 @@ public class AccessionHeaderJson implements AccessionIdentifier3 {
 	public String acceNumb;
 	public String genus;
 
+	public static AccessionHeaderJson fromJson(JsonNode json) throws InvalidDOIException {
+		final AccessionHeaderJson dataJson = new AccessionHeaderJson();
+
+		dataJson.doi = json.has(Api1Constants.Accession.DOI) ? json.get(Api1Constants.Accession.DOI).textValue() : null;
+
+		if (dataJson.doi != null) {
+			DOIUtils.validateDoi(dataJson.doi);
+		}
+
+		dataJson.instCode = json.has(Api1Constants.Accession.INSTCODE) ? json.get(Api1Constants.Accession.INSTCODE).textValue() : null;
+		dataJson.acceNumb = json.has(Api1Constants.Accession.ACCENUMB) ? json.get(Api1Constants.Accession.ACCENUMB).textValue() : null;
+		dataJson.genus = json.has(Api1Constants.Accession.GENUS) ? json.get(Api1Constants.Accession.GENUS).textValue() : null;
+
+		return dataJson;
+	}
+
 	@Override
 	public String getDoi() {
 		return doi;
 	}
-	
+
 	@Override
 	public String getHoldingInstitute() {
 		return instCode;

src/main/resources/dummy-data/IND002.json

+[
+  {
+    "instCode": "IND002",
+    "acceNumb": "IS 1",
+    "acqDate": "19740510",
+    "acqSrc": "40",
+    "available": true,
+    "art15": true,
+    "sampStat": 300,
+    "storage": [12, 13],
+    "origCty": "MEX",
+    "genus": "Sorghum",
+    "species": "bicolor",
+    "spAuthor": "(L.) Moench",
+    "subtaxa": "bicolor",
+    "subtAuthor": null,
+    "cropName": "sorghum"
+  }
+]

src/main/resources/dummy-data/PER001.json

+[{
+  "instCode": "PER001",
+  "acceNumb": "CIP 102081.28",
+  "genus": "Solanum",
+  "cropName": "Sweetpotato",
+  "available": true,
+  "sampStat": 400,
+  "storage": [30],
+  "origCty": "PER"
+}]