Skip to content

GitLab

Genesys Backend
Genesys Backend
Commit 1801cae1 authored by Matija Obreza's avatar Matija Obreza
Browse files
Options

Merge branch '361-institute-numbers' into 'master' 

Resolve "Institute numbers"

Closes #361

See merge request genesys-pgr/genesys-server!263
parents d05dc06f 6642332d
Hide whitespace changes
Inline Side-by-side
Showing with 102 additions and 123 deletions
src/main/java/org/genesys2/server/mvc/WiewsController.java
View file @ 1801cae1
......@@ -35,6 +35,7 @@ import java.util.stream.Stream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.time.StopWatch;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
......@@ -51,6 +52,7 @@ import org.genesys2.server.service.CRMException;
import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.DownloadService;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.ElasticsearchService.TermResult;
import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoService;
......@@ -165,6 +167,7 @@ public class WiewsController extends BaseController {
@RequestMapping("/{wiewsCode}")
public String view(ModelMap model, @PathVariable(value = "wiewsCode") String wiewsCode) {
StopWatch stopWatch = StopWatch.createStarted();
LOG.debug("Viewing institute {}", wiewsCode);
if (!wiewsCode.toUpperCase().equals(wiewsCode)) {
......@@ -177,12 +180,15 @@ public class WiewsController extends BaseController {
}
model.addAttribute("faoInstitute", faoInstitute);
model.addAttribute("organizations", organizationService.getOrganizations(faoInstitute));
LOG.trace("organizations after {}ms", stopWatch.getTime());
model.addAttribute("blurp", contentService.getArticle(faoInstitute, "blurp", getLocale()));
LOG.trace("blurp after {}ms", stopWatch.getTime());
model.addAttribute("countByInstitute", faoInstitute.getAccessionCount());
LOG.trace("countByInstitute after {}ms", stopWatch.getTime());
long datasetCount = genesysService.countDatasets(faoInstitute);
model.addAttribute("datasetCount", datasetCount);
LOG.trace("datasetCount after {}ms", stopWatch.getTime());
{
AccessionFilter af=new AccessionFilter();
......@@ -190,21 +196,29 @@ public class WiewsController extends BaseController {
af.historic = false;
try {
model.addAttribute("statisticsCrops", elasticService.termStatisticsAuto(Accession.class, af, 5, AppliedFiltersConverter
Map<String, TermResult> esStats = elasticService.termStatisticsAuto(Accession.class, af, 5, AppliedFiltersConverter
.convertTerm(FilterConstants.CROPS), AppliedFiltersConverter
.convertTerm(FilterConstants.CROPNAME), AppliedFiltersConverter
.convertTerm(FilterConstants.TAXONOMY_GENUS));
LOG.trace("ES statistics after {}ms", stopWatch.getTime());
model.addAttribute("statisticsCrops", esStats.get(AppliedFiltersConverter
.convertTerm(FilterConstants.CROPS)));
model.addAttribute("statisticsCropName", esStats.get(AppliedFiltersConverter
.convertTerm(FilterConstants.CROPNAME)));
model.addAttribute("statisticsGenus", esStats.get(AppliedFiltersConverter
.convertTerm(FilterConstants.TAXONOMY_GENUS)));
} catch (SearchException e) {
LOG.warn(e.getMessage());
}
}
model.addAttribute("statisticsCropName", genesysService.statisticsCropNameByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsGenus", genesysService.statisticsGenusByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsTaxonomy", genesysService.statisticsSpeciesByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsTaxonomy", genesysService.statisticsSpeciesByInstituteES(faoInstitute, 5));
LOG.trace("statisticsTaxonomy after {}ms", stopWatch.getTime());
model.addAttribute("statisticsPDCI", faoInstitute.getStatisticsPDCI());
LOG.trace("statisticsPDCI after {}ms", stopWatch.getTime());
model.addAttribute("updates", genesysService.getLastUpdatedStatistics(faoInstitute));
if (datasetCount > 0) {
model.addAttribute("statisticsPheno", statisticsService.statisticsPheno(faoInstitute));
}
LOG.trace("updates after {}ms", stopWatch.getTime());
return "/wiews/details";
}
......
src/main/java/org/genesys2/server/service/GenesysService.java
View file @ 1801cae1
......@@ -40,7 +40,6 @@ import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.genesys.PDCI;
import org.genesys2.server.model.genesys.PDCIStatistics;
import org.genesys2.server.model.genesys.PhenoStatistics;
import org.genesys2.server.model.genesys.SvalbardDeposit;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.AccessionIdentifier3;
......@@ -121,6 +120,8 @@ public interface GenesysService {
Page<Object[]> statisticsSpeciesByInstitute(FaoInstitute faoInstitute, Pageable pageable);
List<Object[]> statisticsSpeciesByInstituteES(FaoInstitute faoInstitute, int size);
// Page<Object[]> statisticsCropByInstitute(FaoInstitute faoInstitute,
// Pageable pageable);
......@@ -231,8 +232,6 @@ public interface GenesysService {
PDCIStatistics statisticsPDCI(Organization organization);
PhenoStatistics statisticsPheno(FaoInstitute faoInstitute);
void regenerateAccessionSequentialNumber();
List<SvalbardDeposit> saveSvalbards(List<SvalbardDeposit> svalbards);
......
src/main/java/org/genesys2/server/service/StatisticsService.java
View file @ 1801cae1
......@@ -37,6 +37,4 @@ public interface StatisticsService {
PDCIStatistics statisticsPDCI(Organization organization);
PhenoStatistics statisticsPheno(FaoInstitute faoInstitute);
}
src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
View file @ 1801cae1
......@@ -22,24 +22,47 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.persistence.EntityManager;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import org.apache.commons.collections4.CollectionUtils;
import org.genesys.blocks.security.SecurityContextUtil;
import org.genesys.blocks.security.model.AclSid;
import org.genesys.blocks.security.service.CustomAclService;
import org.genesys2.server.api.model.AccessionHeaderJson;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.genesys.*;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionAlias.AliasType;
import org.genesys2.server.model.genesys.AccessionData;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.AccessionHistoric;
import org.genesys2.server.model.genesys.AccessionId;
import org.genesys2.server.model.genesys.AccessionTrait;
import org.genesys2.server.model.genesys.AllAccnames;
import org.genesys2.server.model.genesys.ExperimentAccessionTrait;
import org.genesys2.server.model.genesys.ExperimentTrait;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.genesys.PDCI;
import org.genesys2.server.model.genesys.PDCIStatistics;
import org.genesys2.server.model.genesys.SelfCopy;
import org.genesys2.server.model.genesys.SvalbardDeposit;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.genesys.TraitCode;
import org.genesys2.server.model.impl.AccessionIdentifier3;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
......@@ -63,11 +86,13 @@ import org.genesys2.server.persistence.SvalbardRepository;
import org.genesys2.server.persistence.TraitValueRepository;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.DatasetService;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.HtmlSanitizer;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.util.NumberUtils;
import org.slf4j.Logger;
......@@ -89,9 +114,11 @@ import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import com.google.common.collect.Sets;
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelper;
import com.opencsv.ResultSetHelperService;
import com.querydsl.jpa.impl.JPAQueryFactory;
@Service
@Transactional(readOnly = true)
......@@ -157,6 +184,9 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Autowired
private GenesysFilterService filterService;
@Autowired
private ElasticsearchService elasticService;
private List<String> availableDisplayColumns;
private List<String> defaultDisplayColumns;
......@@ -647,6 +677,31 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return page;
}
@Override
public List<Object[]> statisticsSpeciesByInstituteES(FaoInstitute institute, int size) {
AccessionFilter af = new AccessionFilter();
af.holder().code = Sets.newHashSet(institute.getCode());
af.historic = false;
ElasticsearchService.TermResult result = null;
try {
result = elasticService.termStatisticsAuto(Accession.class, af, size, "taxonomy.taxSpecies");
} catch (SearchException e) {
LOG.warn(e.getMessage());
}
List<Object[]> statistics = new ArrayList<>(size);
if (result != null) {
for(ElasticsearchService.Term term: result.getTerms()) {
Object[] statItem = new Object[2];
statItem[0] = taxonomyService.get(Long.valueOf(term.getTerm()));
statItem[1] = term.getCount();
statistics.add(statItem);
}
}
return statistics;
}
@Override
public List<Long> listAccessionsIds(Pageable pageable) {
return accessionRepository.listAccessionsIds(pageable);
......@@ -1418,43 +1473,6 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return stats;
}
@Override
public PhenoStatistics statisticsPheno(FaoInstitute faoInstitute) {
List<Object[]> metadatas = accessionTraitRepository.statisticsAccessionMetadata(faoInstitute);
List<Object[]> methods = accessionTraitRepository.statisticsAccessionMethod(faoInstitute);
long count = countByInstitute(faoInstitute);
PhenoStatistics stats = new PhenoStatistics();
stats.setCount(count);
{
long withMetadata = 0, metadatasTotal = 0;
for (Object[] m : metadatas) {
// Long accessionId = (Long) m[0];
Long metadataCount = (Long) m[1];
if (metadataCount != null && metadataCount > 0) {
withMetadata++;
metadatasTotal += metadataCount;
}
}
stats.setWithMetadata(withMetadata);
stats.setAvgMetadatas((float) metadatasTotal / withMetadata);
}
{
long withMethod = 0, methodsTotal = 0;
for (Object[] m : methods) {
// Long accessionId = (Long) m[0];
Long methodCount = (Long) m[1];
if (methodCount != null && methodCount > 0) {
withMethod++;
methodsTotal += methodCount;
}
}
stats.setWithMethod(withMethod);
stats.setAvgMethods((float) methodsTotal / withMethod);
}
return stats;
}
@Transactional(readOnly = false)
@Override
......
src/main/java/org/genesys2/server/service/impl/StatisticsServiceImpl.java
View file @ 1801cae1
......@@ -21,8 +21,6 @@ import org.genesys.catalog.service.DatasetService;
import org.genesys2.server.model.PublishState;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.PDCIStatistics;
import org.genesys2.server.model.genesys.PhenoStatistics;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.Organization;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.GenesysService;
......@@ -109,15 +107,4 @@ public class StatisticsServiceImpl implements StatisticsService {
}
}
@Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-' + #faoInstitute.id")
public PhenoStatistics statisticsPheno(FaoInstitute faoInstitute) {
synchronized (this) {
if (LOG.isDebugEnabled()) {
LOG.debug("Regenerating C&E statistics for {}", faoInstitute);
}
return genesysService.statisticsPheno(faoInstitute);
}
}
}
src/main/java/org/genesys2/spring/config/ApplicationStartup.java
View file @ 1801cae1
......@@ -26,10 +26,6 @@ import org.genesys.blocks.security.UserException;
import org.genesys.blocks.security.model.BasicUser;
import org.genesys.blocks.security.service.CustomAclService;
import org.genesys.blocks.security.service.PasswordPolicy.PasswordPolicyException;
import org.genesys.catalog.persistence.PartnerRepository;
import org.genesys.catalog.persistence.dataset.DatasetRepository;
import org.genesys.catalog.persistence.traits.DescriptorListRepository;
import org.genesys.catalog.persistence.traits.DescriptorRepository;
import org.genesys2.server.model.UserRole;
import org.genesys2.server.model.impl.User;
import org.genesys2.server.persistence.UserRepository;
......@@ -99,12 +95,6 @@ public class ApplicationStartup implements InitializingBean, ApplicationListener
}
startup();
asAdminInvoker.invoke(() -> {
aclForCatalog();
// aclParentObject();
return null;
});
}
/**
......@@ -178,32 +168,6 @@ public class ApplicationStartup implements InitializingBean, ApplicationListener
LOG.info("Startup initializer done.");
}
@Autowired
private PartnerRepository partnerRepository;
@Autowired
private DatasetRepository datasetRepository;
@Autowired
private DescriptorRepository descriptorRepository;
@Autowired
private DescriptorListRepository descriptorListRepository;
// TODO Remove after first deployment to sandbox
private void aclForCatalog() {
partnerRepository.findAll().stream().forEach(partner -> aclService.addCreatorPermissions(partner));
// datasetRepository.findAll().stream().forEach(dataset ->
// aclService.removeAclAwareModel(dataset));
datasetRepository.findAll().stream().forEach(dataset -> aclService.addCreatorPermissions(dataset));
// descriptorListRepository.findAll().stream().forEach(descriptorList ->
// aclService.removeAclAwareModel(descriptorList));
descriptorListRepository.findAll().stream().forEach(descriptorList -> aclService.addCreatorPermissions(descriptorList));
// descriptorRepository.findAll().stream().forEach(descriptor ->
// aclService.removeAclAwareModel(descriptor));
descriptorRepository.findAll().stream().forEach(descriptor -> aclService.addCreatorPermissions(descriptor));
}
@Autowired
private ElasticsearchService elasticsearch;
......
src/main/webapp/WEB-INF/jsp/wiews/details.jsp
View file @ 1801cae1
......@@ -334,16 +334,16 @@
</div>
</c:if>
<c:if test="${statisticsCropName ne null and statisticsCropName.numberOfElements gt 0}">
<c:if test="${statisticsCropName ne null and statisticsCropName.terms.size() gt 0}">
<div class="collect-info col-lg-6 col-md-12 col-sm-12 col-xs-12 stat-by-genus">
<h4 class="row section-heading"><span><spring:message code="faoInstitute.stat-by-cropName" arguments="${statisticsGenus.numberOfElements}" /></span></h4>
<h4 class="row section-heading"><span><spring:message code="faoInstitute.stat-by-cropName" arguments="${statisticsGenus.terms.size()}" /></span></h4>
<div class="section-inner-content clearfix ${cssClass}">
<div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 statistics">
<c:forEach items="${statisticsCropName.content}" var="stat" varStatus="status">
<c:forEach items="${statisticsCropName.terms}" var="stat" varStatus="status">
<div class="row">
<div class="col-lg-8 col-md-6 col-sm-6 col-xs-6 ${status.count % 2 == 0 ? 'even' : 'odd'}">
<c:choose>
<c:when test="${stat[0] eq null}">
<c:when test="${stat.term eq null}">
<span>
<em>
<spring:message code="data-overview.missingCount" />
......@@ -351,17 +351,17 @@
</span>
</c:when>
<c:otherwise>
<a title="<spring:message code="faoInstitute.link-species-data" arguments="${faoInstitute.fullName}|${faoInstitute.acronym}|${stat[0]}" argumentSeparator="|" />"
href="<c:url value="/wiews/${faoInstitute.code}/cropName/${stat[0]}" />">
<a title="<spring:message code="faoInstitute.link-species-data" arguments="${faoInstitute.fullName}|${faoInstitute.acronym}|${stat.term}" argumentSeparator="|" />"
href="<c:url value="/wiews/${faoInstitute.code}/cropName/${stat.term}" />">
<span dir="ltr">
<c:out value="${stat[0]}" />
<c:out value="${stat.term}" />
</span>
</a>
</c:otherwise>
</c:choose>
</div>
<div class="col-lg-4 col-md-6 col-sm-6 col-xs-6 ${status.count % 2 == 0 ? 'even' : 'odd'}">
<span class="stats-number"><fmt:formatNumber value="${stat[1]}" /></span>
<span class="stats-number"><fmt:formatNumber value="${stat.count}" /></span>
</div>
</div>
</c:forEach>
......@@ -375,22 +375,21 @@
</div>
</c:if>
<c:if test="${statisticsGenus ne null and statisticsGenus.numberOfElements gt 0}">
<c:if test="${statisticsGenus ne null and statisticsGenus.terms.size() gt 0}">
<div class="collect-info col-lg-6 col-md-12 col-sm-12 col-xs-12 stat-by-genus">
<h4 class="row section-heading"><span><spring:message code="faoInstitute.stat-by-genus" arguments="${statisticsGenus.numberOfElements}" /></span></h4>
<h4 class="row section-heading"><span><spring:message code="faoInstitute.stat-by-genus" arguments="${statisticsGenus.terms.size()}" /></span></h4>
<div class="section-inner-content clearfix ${cssClass}">
<div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 statistics">
<c:forEach items="${statisticsGenus.content}" var="stat" varStatus="status">
<c:forEach items="${statisticsGenus.terms}" var="stat" varStatus="status">
<div class="row">
<div class="col-lg-8 col-md-6 col-sm-6 col-xs-6 ${status.count % 2 == 0 ? 'even' : 'odd'}">
<%-- <span class="index-number">${status.count + pagedData.size * pagedData.number}</span> --%>
<a title="<spring:message code="faoInstitute.link-species-data" arguments="${faoInstitute.fullName}|${faoInstitute.acronym}|${stat[0]}" argumentSeparator="|" />" href="<c:url value="/wiews/${faoInstitute.code}/t/${stat[0]}" />">
<span dir="ltr" class="sci-name"><i><c:out value="${stat[0]}" /></i></span>
<a title="<spring:message code="faoInstitute.link-species-data" arguments="${faoInstitute.fullName}|${faoInstitute.acronym}|${stat.term}" argumentSeparator="|" />" href="<c:url value="/wiews/${faoInstitute.code}/t/${stat.term}" />">
<span dir="ltr" class="sci-name"><i><c:out value="${stat.term}" /></i></span>
</a>
</div>
<div class="col-lg-4 col-md-6 col-sm-6 col-xs-6 ${status.count % 2 == 0 ? 'even' : 'odd'}">
<span class="stats-number"><fmt:formatNumber value="${stat[1]}" /></span>
<span class="stats-number"><fmt:formatNumber value="${stat.count}" /></span>
</div>
</div>
</c:forEach>
......@@ -404,12 +403,12 @@
</div>
</c:if>
<c:if test="${statisticsTaxonomy ne null and statisticsTaxonomy.numberOfElements gt 0}">
<c:if test="${statisticsTaxonomy ne null and statisticsTaxonomy.size() gt 0}">
<div class="collect-info col-lg-6 col-md-12 col-sm-12 col-xs-12 stat-by-species">
<h4 class="row section-heading"><span><spring:message code="faoInstitute.stat-by-species" arguments="${statisticsTaxonomy.numberOfElements}" /></span></h4>
<h4 class="row section-heading"><span><spring:message code="faoInstitute.stat-by-species" arguments="${statisticsTaxonomy.size()}" /></span></h4>
<div class="section-inner-content clearfix ${cssClass}">
<div class="col-lg-8 col-md-8 col-sm-8 col-xs-12 statistics">
<c:forEach items="${statisticsTaxonomy.content}" var="stat" varStatus="status">
<c:forEach items="${statisticsTaxonomy}" var="stat" varStatus="status">
<div class="row">
<div class="col-lg-8 col-md-6 col-sm-6 col-xs-6 ${status.count % 2 == 0 ? 'even' : 'odd'}">
<%-- <span class="index-number">${status.count + pagedData.size * pagedData.number}</span> --%>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment