Skip to content

GitLab

Commit 8f48536d authored by Matija Obreza's avatar Matija Obreza
Browse files

Export repository metadata reviewed

parent 5834281c
Hide whitespace changes
Inline Side-by-side
Showing with 205 additions and 169 deletions
src/main/java/org/genesys2/server/service/GenesysService.java
View file @ 8f48536d
......@@ -292,12 +292,4 @@ public interface GenesysService {
Page<AccessionDetails> listAccessionsDetails(Collection<Long> accessionIds, Pageable page);
/**
* Method for download files metadata
* @param response response
* @param repositoryPath repository path
* @throws IOException
*/
void downloadFilesMetadata(HttpServletResponse response, String repositoryPath) throws IOException;
}
src/main/java/org/genesys2/server/service/impl/ExporterToCSV.java 0 → 100644
View file @ 8f48536d
/*
* Copyright 2017 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.service.impl;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.stream.Stream;
import javax.servlet.http.HttpServletResponse;
import com.opencsv.CSVWriter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* Writes data in CSV format
*
* @author Andrey Lugovskoy.
*/
public abstract class ExporterToCSV<T> {
public static final Log LOG = LogFactory.getLog(ExporterToCSV.class);
public void downloadMetadata(Stream<T> entities, HttpServletResponse response, char separator, char quoteChar, char escapeChar, String lineEnd, String encoding) throws IOException {
try (CSVWriter writer = new CSVWriter(new BufferedWriter(new OutputStreamWriter(response.getOutputStream(), encoding)), separator, quoteChar, escapeChar, lineEnd)) {
writer.writeNext(writeHeaders());
entities.forEach(element -> {
writer.writeNext(prepareLine(element), true);
});
writer.flush();
}
}
protected abstract String[] writeHeaders();
protected abstract String[] prepareLine(T element);
}
src/main/java/org/genesys2/server/service/impl/FilesMetadataInfo.java
View file @ 8f48536d
/*
* Copyright 2017 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.service.impl;
import org.genesys.filerepository.model.RepositoryFile;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.List;
import org.springframework.stereotype.Component;
/**
* @author Andrey Lugovskoy.
*/
@Service("filesMetadataInfo")
public class FilesMetadataInfo extends MetadataInfo<RepositoryFile> {
@Component
public class FilesMetadataInfo extends ExporterToCSV<RepositoryFile> {
@Value("#{'${files.metadata.fields}'.split(',')}") private List<String> filesMetadataHeader;
// @formatter:off
private final String[] CSV_HEADERS = new String[] {
"uuid",
"version",
"contentType",
"path",
"extension",
"originalFilename",
"title",
"subject",
"description",
"creator",
"created",
"rightsHolder",
"accessRights",
"license",
"bibliographicCitation"
};
// @formatter:on
@Override
protected String[] writeHeaders() {
return CSV_HEADERS;
}
@Override
public void downloadMetadata(List<RepositoryFile> entities, HttpServletResponse response, char separator, char quoteChar, char escapeChar, String lineEnd) throws IOException {
super.downloadMetadata(filesMetadataHeader, entities, response, separator, quoteChar, escapeChar, lineEnd);
protected String[] prepareLine(RepositoryFile element) {
// @formatter:off
return new String[] {
element.getUuid().toString(),
element.getVersion().toString(),
element.getContentType(),
element.getPath(),
element.getExtension(),
element.getOriginalFilename(),
element.getTitle(),
element.getSubject(),
element.getDescription(),
element.getCreator(),
element.getCreated(),
element.getRightsHolder(),
element.getAccessRights(),
element.getLicense(),
element.getBibliographicCitation()
};
// @formatter:on
}
}
src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
View file @ 8f48536d
......@@ -16,16 +16,37 @@
package org.genesys2.server.service.impl;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.persistence.EntityManager;
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelper;
import com.opencsv.ResultSetHelperService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang.StringUtils;
import org.genesys.blocks.security.SecurityContextUtil;
import org.genesys.blocks.security.service.CustomAclService;
import org.genesys.filerepository.model.RepositoryFile;
import org.genesys.filerepository.service.RepositoryService;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionAlias;
......@@ -106,28 +127,6 @@ import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import javax.persistence.EntityManager;
import javax.servlet.http.HttpServletResponse;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Service
@Transactional(readOnly = true)
public class GenesysServiceImpl implements GenesysService, DatasetService {
......@@ -201,12 +200,6 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Autowired
private GenesysFilterService filterService;
@Autowired(required = false)
private RepositoryService repositoryService;
@Autowired(required = false)
private FilesMetadataInfo filesMetadataInfo;
private List<String> availableDisplayColumns;
private List<String> defaultDisplayColumns;
......@@ -1911,16 +1904,4 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return accessionRepository.lastUpdatedStatistics(faoInstitute);
}
/**
* {@inheritDoc}
*
* @param response
*/
@Override
public void downloadFilesMetadata(HttpServletResponse response, String repositoryPath) throws IOException {
List<RepositoryFile> files = repositoryService.getFiles(repositoryPath);
filesMetadataInfo.downloadMetadata(files, response, ' ', '"', '\\', "\r\n");
}
}
src/main/java/org/genesys2/server/service/impl/MetadataInfo.java deleted 100644 → 0
View file @ 5834281c
package org.genesys2.server.service.impl;
import com.opencsv.CSVWriter;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import javax.servlet.http.HttpServletResponse;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
/**
* @author Andrey Lugovskoy.
*/
public abstract class MetadataInfo<T> {
public static final Log LOG = LogFactory.getLog(MetadataInfo.class);
public abstract void downloadMetadata(List<T> entities, HttpServletResponse response, char separator, char quoteChar, char escapeChar, String lineEnd) throws IOException;
public void downloadMetadata(List<String> filesMetadataFields, List<T> entities, HttpServletResponse response, char separator, char quoteChar, char escapeChar, String lineEnd)
throws IOException {
try (CSVWriter writer = new CSVWriter(new BufferedWriter(new OutputStreamWriter(response.getOutputStream())), separator, quoteChar, escapeChar, lineEnd)) {
writer.writeNext(filesMetadataFields.toArray(ArrayUtils.EMPTY_STRING_ARRAY));
entities.stream().forEach(element -> {
List<Object> values = new ArrayList<>();
Class<?> aClass = element.getClass();
filesMetadataFields.stream().forEach(fieldName -> {
Field field;
Object value = " ";
try {
field = aClass.getField(fieldName);
value = field.get(element);
} catch (NoSuchFieldException e) {
try {
field = aClass.getSuperclass().getDeclaredField(fieldName);
field.setAccessible(true);
value = field.get(element);
value = value == null ? " " : value;
} catch (NoSuchFieldException | IllegalAccessException e1) {
LOG.error("Error getting value from field in " + element.getClass().getName());
}
} catch (IllegalAccessException e) {
LOG.error("Error getting value from field in " + element.getClass().getName());
}
values.add(value);
});
String[] stockArr = new String[values.size()];
stockArr = values.toArray(stockArr);
writer.writeNext(stockArr);
});
writer.flush();
}
}
}
src/main/java/org/genesys2/server/servlet/controller/WiewsController.java
View file @ 8f48536d
......@@ -16,6 +16,23 @@
package org.genesys2.server.servlet.controller;
import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Stream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
......@@ -36,6 +53,7 @@ import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.StatisticsService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.impl.FilesMetadataInfo;
import org.genesys2.server.service.impl.FilterHandler;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilter;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
......@@ -50,6 +68,7 @@ import org.springframework.data.domain.Sort;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.WebDataBinder;
import org.springframework.web.bind.annotation.GetMapping;
......@@ -66,27 +85,12 @@ import org.springframework.web.servlet.HandlerMapping;
import org.springframework.web.servlet.mvc.support.RedirectAttributes;
import org.springframework.web.util.UriUtils;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@Controller
@Scope("request")
@RequestMapping("/wiews")
public class WiewsController extends BaseController {
private static final String MANAGE_FILES_JSP_PATH = "/wiews/files";
private static final String MANAGE_FILES_JSP_PATH = "/wiews/files";
@Autowired
private InstituteService instituteService;
......@@ -115,11 +119,15 @@ public class WiewsController extends BaseController {
@Autowired
private ElasticService elasticService;
@Autowired
private RepositoryService repositoryService;
@Autowired
private RepositoryService repositoryService;
@Autowired
private ImageGalleryService imageGalleryService;
@Autowired(required = false)
private FilesMetadataInfo filesMetadataInfo;
@Autowired
private ImageGalleryService imageGalleryService;
@RequestMapping("/")
public String view(ModelMap model, @RequestParam(value = "page", required = false, defaultValue = "1") int page) {
......@@ -597,7 +605,6 @@ public class WiewsController extends BaseController {
instituteService.getInstituteForEdit(wiewsCode);
ImageGallery imageGallery = imageGalleryService.loadImageGallery(galleryPath);
String gallerySubPath = galleryPath.replace("/wiews/" + wiewsCode, "");
if (imageGallery == null) {
throw new ResourceNotFoundException("No image gallery here!");
......@@ -656,20 +663,24 @@ public class WiewsController extends BaseController {
}
@RequestMapping(value = "{wiewsCode}/files/download/metadata")
@Transactional(readOnly = true)
public void downloadMetadata(@PathVariable("wiewsCode") String wiewsCode, HttpServletResponse response) throws IOException {
// check user permissions
instituteService.getInstituteForEdit(wiewsCode);
final String repositoryPath = "/wiews/" + wiewsCode;
response.setHeader("Content-Disposition", "attachment; filename=" + wiewsCode +"_files_metadata.csv ");
genesysService.downloadFilesMetadata(response, repositoryPath);
response.setContentType("text/csv;charset=UTF-16LE");
response.setHeader("Content-Disposition", "attachment; filename=" + wiewsCode + "_files_metadata.csv ");
Stream<RepositoryFile> files = repositoryService.streamFiles(repositoryPath);
filesMetadataInfo.downloadMetadata(files, response, '\t', '"', '\\', "\n", "UTF-16LE");
}
@InitBinder
public void initBinder(WebDataBinder binder) {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
dateFormat.setLenient(false);
binder.registerCustomEditor(Date.class, new CustomDateEditor(dateFormat, false));
}
public void initBinder(WebDataBinder binder) {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
dateFormat.setLenient(false);
binder.registerCustomEditor(Date.class, new CustomDateEditor(dateFormat, false));
}
}
src/main/java/org/genesys2/spring/config/ApplicationConfig.java
View file @ 8f48536d
......@@ -63,7 +63,6 @@ public class ApplicationConfig {
final List<Resource> locations = new ArrayList<>();
locations.add(new ClassPathResource("application.properties"));
locations.add(new ClassPathResource("files-metadata.properties"));
locations.add(new ClassPathResource("genesys.properties"));
final String extraConfigFile = System.getenv("config.file");
......
src/main/resources/application.properties
View file @ 8f48536d
......@@ -43,6 +43,14 @@ base.cookie-http-only=true
# robots.txt
robots.allow=false
db.url=jdbc:mysql://localhost/genesys?useUnicode=true&characterEncoding=UTF-8&useFastDateParsing=false
db.driverClassName=com.mysql.jdbc.Driver
db.username=root
db.password=
db.showSql=false
db.hbm2ddl=false
hibernate.dialect=org.hibernate.dialect.MySQL5InnoDBDialect
google.api.key=AIzaSyBEvPuc8j5ps5GDQ3tdnTJaffKhfOdxFVc
google.url.shortener=https://www.googleapis.com/urlshortener/v1/url
news.menu.abbreviate.length=22
......
src/main/resources/content/language.properties
View file @ 8f48536d
......@@ -853,7 +853,7 @@ repository.file.title=Title
repository.file.subject=Subject
repository.file.description=Description
repository.file.creator=Creator
repository.file.created=Created date
repository.file.created=Resource created
repository.file.rightsHolder=Rights holder
repository.file.accessRights=Access rights
repository.file.license=License name or URL
......@@ -864,6 +864,7 @@ repository.file.extent=Extent
repository.file.bibliographicCitation=Bibliographic citation
repository.file.dateSubmitted=Date of submission
repository.file.lastModifiedDate=Last modified date
repository.file.createdDate=Date of creation
file.upload-file=Upload file
file.download-metadata=Download metadata
......
src/main/resources/content/language_ar.properties
View file @ 8f48536d
......@@ -859,5 +859,4 @@ welcome.networks=الشبكات
geo.country=الدولة:
file.download-metadata=تنزيل البيانات الوصفية
file.download-metadata=تنزيل البيانات الوصفية
article.is.template=هو القالب:
src/main/resources/content/language_de.properties
View file @ 8f48536d
......@@ -859,5 +859,4 @@ welcome.networks=Netzwerke
geo.country=Land:
file.download-metadata=Metadaten herunterladen
file.download-metadata=Metadaten herunterladen
article.is.template=Ist Vorlage:
src/main/resources/content/language_en.properties
View file @ 8f48536d
......@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#-------------------------------------------------------------------------------
file.download-metadata=Download metadata
src/main/resources/content/language_es.properties
View file @ 8f48536d
......@@ -859,5 +859,4 @@ welcome.networks=Redes
geo.country=País:
file.download-metadata=Descargar metadatos
file.download-metadata=Descargar metadatos
article.is.template=Es plantilla:
src/main/resources/content/language_fa.properties
View file @ 8f48536d
......@@ -859,5 +859,4 @@ welcome.networks=شبکه‌ها
geo.country=کشور:
file.download-metadata=دانلود ابرداده
file.download-metadata=دانلود ابرداده
article.is.template=الگو است:
src/main/resources/content/language_fr.properties
View file @ 8f48536d
......@@ -859,5 +859,4 @@ welcome.networks=Réseaux
geo.country=Pays :
file.download-metadata=Télécharger les métadonnées
file.download-metadata=Télécharger les métadonnées
article.is.template=Est-ce que le modèle:
src/main/resources/content/language_pt.properties
View file @ 8f48536d
......@@ -859,5 +859,4 @@ welcome.networks=Redes
geo.country=País:
file.download-metadata=Baixar metadados
file.download-metadata=Baixar metadados
article.is.template=É modelo:
src/main/resources/content/language_ru.properties
View file @ 8f48536d
......@@ -859,5 +859,4 @@ welcome.networks=Сети
geo.country=Страна:
file.download-metadata=Загрузить метаданные
file.download-metadata=Загрузить метаданные
article.is.template=Шаблон:
src/main/resources/content/language_zh.properties
View file @ 8f48536d
......@@ -859,5 +859,4 @@ welcome.networks=网站
geo.country=国家/地区:
file.download-metadata=下載元數據
file.download-metadata=下載元數據
article.is.template=是模板
src/main/resources/files-metadata.properties deleted 100644 → 0
View file @ 5834281c
files.metadata.fields=originalFilename,extension,title,subject,description,creator,created,license,contentType
src/main/webapp/WEB-INF/jsp/wiews/files/edit.jsp
View file @ 8f48536d
......@@ -47,7 +47,7 @@
<div class="col-md-6 margin-top-20">
<label for="created"><spring:message code="repository.file.created"/></label>
<input type="text" id="created" name="created" class="form-control" value="${file.createdDate}"
<input type="text" id="created" name="created" class="form-control" value="${file.created}"
placeholder="<spring:message code="repository.file.created" />">
</div>
......@@ -97,14 +97,22 @@
<div class="col-md-6 margin-top-20">
<label for="dateSubmitted"><spring:message code="repository.file.dateSubmitted"/></label>
<input type="text" id="dateSubmitted" name="createdDate" class="form-control" value="${file.dateSubmitted}"
<input type="text" id="dateSubmitted" name="dateSubmitted" class="form-control" value="${file.dateSubmitted}"
placeholder="<spring:message code="repository.file.dateSubmitted" />">
</div>
<div class="col-md-6 margin-top-20">
<label for="modified"><spring:message code="repository.file.lastModifiedDate"/></label>
<input type="text" id="modified" name="lastModifiedDate" class="form-control" value="${file.modified}"
placeholder="<spring:message code="repository.file.lastModifiedDate" />">
<label for="createdDate"><spring:message code="repository.file.createdDate"/></label>
<p type="text" id="createdDate" name="createdDate" class="form-control-static">
<local:prettyTime date="${file.createdDate}" locale="${pageContext.response.locale}"/>
</p>
</div>
<div class="col-md-6 margin-top-20">
<label for="lastModifiedDate"><spring:message code="repository.file.lastModifiedDate"/></label>
<p type="text" id="lastModifiedDate" name="lastModifiedDate" class="form-control-static">
<local:prettyTime date="${file.lastModifiedDate}" locale="${pageContext.response.locale}"/>
</p>