Commit 53c2cffb authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch '119-repository-files' into 'master'

Resolve "Repository files"

Closes #119

See merge request !110
parents 42c9a0be 3423d9eb
......@@ -165,8 +165,8 @@ public interface DatasetService {
* @throws InvalidRepositoryPathException InvalidRepositoryPathException
* @throws InvalidRepositoryFileDataException InvalidRepositoryFileDataException
*/
@PreAuthorize("isAuthenticated()")
Dataset addDatasetFile(UUID datasetUuid, MultipartFile file) throws NotFoundElement, IOException, InvalidRepositoryPathException, InvalidRepositoryFileDataException;
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#dataset, 'write')")
Dataset addDatasetFile(Dataset dataset, MultipartFile file) throws NotFoundElement, IOException, InvalidRepositoryPathException, InvalidRepositoryFileDataException;
/**
* Removes the file of dataset.
......@@ -178,8 +178,8 @@ public interface DatasetService {
* @throws NoSuchRepositoryFileException NoSuchRepositoryFileException
* @throws IOException IOException
*/
@PreAuthorize("isAuthenticated()")
Dataset removeDatasetFile(UUID datasetUuid, UUID fileUuid) throws NotFoundElement, NoSuchRepositoryFileException, IOException;
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#dataset, 'write')")
Dataset removeDatasetFile(Dataset dataset, UUID fileUuid) throws NotFoundElement, NoSuchRepositoryFileException, IOException;
/**
* Load list of RepositoryFile by uuid of dataset.
......@@ -188,7 +188,7 @@ public interface DatasetService {
* @return loaded list of RepositoryFile
* @throws NotFoundElement the not found element
*/
List<RepositoryFile> listDatasetFiles(UUID datasetUuid) throws NotFoundElement;
List<RepositoryFile> listDatasetFiles(Dataset dataset) throws NotFoundElement;
/**
* Remove dataset.
......
......@@ -16,7 +16,7 @@
package org.genesys.catalog.service.impl;
import java.io.IOException;
import java.time.Instant;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
......@@ -24,7 +24,6 @@ import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.genesys.catalog.exceptions.InvalidApiUsageException;
import org.genesys.catalog.exceptions.NotFoundElement;
......@@ -78,8 +77,8 @@ public class DatasetServiceImpl implements DatasetService {
private Utils utils;
/** The file repository path. */
@Value("${file.repository.path}")
public String fileRepositoryPath;
@Value("${file.repository.datasets.folder}")
public String datasetRepositoryPath;
/**
* {@inheritDoc}
......@@ -278,10 +277,12 @@ public class DatasetServiceImpl implements DatasetService {
*/
@Transactional
@Override
public Dataset addDatasetFile(final UUID datasetUuid, final MultipartFile file) throws NotFoundElement, IOException, InvalidRepositoryPathException,
public Dataset addDatasetFile(Dataset dataset, final MultipartFile file) throws NotFoundElement, IOException, InvalidRepositoryPathException,
InvalidRepositoryFileDataException {
final Dataset dataset = loadDataset(datasetUuid);
final RepositoryFile repositoryFile = fileRepoService.addFile(fileRepositoryPath, buildFileName(file.getOriginalFilename()), file.getContentType(), file.getBytes(), null);
dataset = datasetRepository.findByUuidAndVersion(dataset.getUuid(), dataset.getVersion());
final RepositoryFile repositoryFile = fileRepoService.addFile(Paths.get(datasetRepositoryPath, dataset.getUuid().toString()).toAbsolutePath().toString(), file
.getOriginalFilename(), file.getContentType(), file.getBytes(), null);
dataset.getRepositoryFiles().add(repositoryFile);
return lazyLoad(datasetRepository.save(dataset));
}
......@@ -291,8 +292,9 @@ public class DatasetServiceImpl implements DatasetService {
*/
@Transactional
@Override
public Dataset removeDatasetFile(final UUID datasetUuid, final UUID fileUuid) throws NotFoundElement, NoSuchRepositoryFileException, IOException {
final Dataset dataset = loadDataset(datasetUuid);
public Dataset removeDatasetFile(Dataset dataset, final UUID fileUuid) throws NotFoundElement, NoSuchRepositoryFileException, IOException {
dataset = datasetRepository.findByUuidAndVersion(dataset.getUuid(), dataset.getVersion());
final RepositoryFile repositoryFile = fileRepoService.getFile(fileUuid);
dataset.getRepositoryFiles().remove(repositoryFile);
fileRepoService.removeFile(repositoryFile);
......@@ -303,8 +305,7 @@ public class DatasetServiceImpl implements DatasetService {
* {@inheritDoc}
*/
@Override
public List<RepositoryFile> listDatasetFiles(final UUID datasetUuid) throws NotFoundElement {
final Dataset dataset = loadDataset(datasetUuid);
public List<RepositoryFile> listDatasetFiles(final Dataset dataset) throws NotFoundElement {
return dataset.getRepositoryFiles();
}
......@@ -430,19 +431,4 @@ public class DatasetServiceImpl implements DatasetService {
target.getDescriptors().addAll(descriptors.stream().distinct().collect(Collectors.toList()));
}
/**
* Builds the file name.
*
* @param originName the origin name
* @return the string
*/
private String buildFileName(final String originName) {
final Instant instant = Instant.now();
final String[] s = originName.split("[.]");
final StringBuilder stringBuilder = new StringBuilder();
IntStream.range(0, s.length - 1).forEach(i -> stringBuilder.append(s[i]));
stringBuilder.append("_").append(instant).append(".").append(s[s.length - 1]);
return stringBuilder.toString();
}
}
......@@ -14,7 +14,11 @@
# limitations under the License.
#-------------------------------------------------------------------------------
file.repository.path=/file_repository
# Directory for FileSystemStorageService
file.repository.dir=data/file_repository
# Path to dataset files within the repository
file.repository.datasets.folder=/dataset
#AWS parameter values
s3.accessKey=
s3.secretKey=
......
......@@ -15,13 +15,7 @@
*/
package org.genesys.catalog.service;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertThat;
import java.io.File;
......@@ -35,8 +29,6 @@ import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import com.google.common.collect.Lists;
import org.genesys.blocks.model.filters.StringFilter;
import org.genesys.catalog.exceptions.InvalidApiUsageException;
import org.genesys.catalog.exceptions.NotFoundElement;
......@@ -59,6 +51,8 @@ import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.mock.web.MockMultipartFile;
import com.google.common.collect.Lists;
public class DatasetServiceTest extends AbstractDatasetServiceTest {
@Test
......@@ -260,7 +254,7 @@ public class DatasetServiceTest extends AbstractDatasetServiceTest {
Dataset savedDataset = buildAndSaveDataset(DATASET_TITLE_1, DATASET_DESCRIPTION_1, partner, false);
savedDataset = datasetService.updateDescriptors(savedDataset, Lists.newArrayList(descriptor3, descriptor1, descriptor2, descriptor3));
assertThat(savedDataset.getDescriptors(), hasSize(3));
}
......@@ -340,12 +334,11 @@ public class DatasetServiceTest extends AbstractDatasetServiceTest {
MockMultipartFile mockMultipartFile = new MockMultipartFile("file", file.getName(), "multipart/form-data", fileInputStream);
Dataset created = buildAndSaveDataset(DATASET_TITLE_1, DATASET_DESCRIPTION_1, partner, false);
Dataset dataset = datasetService.addDatasetFile(created.getUuid(), mockMultipartFile);
Dataset dataset = datasetService.addDatasetFile(created, mockMultipartFile);
File file2 = new File(getClass().getResource("/mcpd20177.csv").getPath());
FileInputStream fileInputStream2 = new FileInputStream(file2);
MockMultipartFile mockMultipartFile2 = new MockMultipartFile("file", file2.getName(), "multipart/form-data", fileInputStream2);
dataset = datasetService.addDatasetFile(created.getUuid(), mockMultipartFile2);
FileInputStream fileInputStream2 = new FileInputStream(file);
MockMultipartFile mockMultipartFile2 = new MockMultipartFile("file", "2" + file.getName(), "multipart/form-data", fileInputStream2);
dataset = datasetService.addDatasetFile(dataset, mockMultipartFile2);
assertThat(dataset.getRepositoryFiles(), is(notNullValue()));
}
......@@ -357,12 +350,12 @@ public class DatasetServiceTest extends AbstractDatasetServiceTest {
Dataset input = buildAndSaveDataset(DATASET_TITLE_1, DATASET_DESCRIPTION_1, partner, false);
Dataset dataset = datasetService.addDatasetFile(input.getUuid(), mockMultipartFile);
input = datasetService.addDatasetFile(input, mockMultipartFile);
datasetService.removeDatasetFile(input.getUuid(), dataset.getRepositoryFiles().get(0).getUuid());
Dataset dataset = datasetService.removeDatasetFile(input, input.getRepositoryFiles().get(0).getUuid());
assertThat(input.getRepositoryFiles().size(), is(0));
assertThat(repositoryFilePersistence.findByUuid(dataset.getRepositoryFiles().get(0).getUuid()), nullValue());
assertThat(dataset.getRepositoryFiles().size(), is(0));
assertThat(repositoryFilePersistence.findByUuid(input.getRepositoryFiles().get(0).getUuid()), nullValue());
}
@Test
......@@ -423,7 +416,8 @@ public class DatasetServiceTest extends AbstractDatasetServiceTest {
filter.accessionIdentifier.genus.clear();
filter.accessionIdentifier.genus("Manihot");
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent(), hasSize(1));
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent().stream().map(d -> d.getUuid()).collect(Collectors.toSet()), contains(dataset2.getUuid()));
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent().stream().map(d -> d.getUuid()).collect(Collectors.toSet()), contains(dataset2
.getUuid()));
filter.accessionIdentifier.genus.clear();
filter.accessionIdentifier.genus("Manihot");
......@@ -444,7 +438,8 @@ public class DatasetServiceTest extends AbstractDatasetServiceTest {
filter.accessionIdentifier.acceNumb.eq = "A8"; // only in dataset1
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent(), hasSize(1));
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent().stream().map(d -> d.getUuid()).collect(Collectors.toSet()), contains(dataset1.getUuid()));
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent().stream().map(d -> d.getUuid()).collect(Collectors.toSet()), contains(dataset1
.getUuid()));
filter.accessionIdentifier.acceNumb.eq = "A1"; // in both
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent(), hasSize(2));
......@@ -452,9 +447,9 @@ public class DatasetServiceTest extends AbstractDatasetServiceTest {
filter.accessionIdentifier.acceNumb.eq = "A8"; // only in dataset1
filter.accessionIdentifier.genus("Musa");
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent(), hasSize(1));
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent().stream().map(d -> d.getUuid()).collect(Collectors.toSet()), contains(dataset1.getUuid()));
assertThat(datasetService.listDatasets(filter, new PageRequest(0, 3)).getContent().stream().map(d -> d.getUuid()).collect(Collectors.toSet()), contains(dataset1
.getUuid()));
}
@Test(expected = InvalidApiUsageException.class)
public void failUpdateDatasetOwner() throws Exception {
......
......@@ -39,7 +39,10 @@ db.updateSchema=true
# Default partner UUID
partner.primary.uuid=a870b3b2-1dfc-459f-abc8-8c10ca8aaab4
file.repository.path=/file_repository
# Directory for FileSystemStorageService
file.repository.dir=data/file_repository
# Path to dataset files within the repository
file.repository.datasets.folder=/dataset
#AWS parameter values
s3.accessKey=
......
......@@ -20,38 +20,25 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.genesys.blocks.util.CurrentApplicationContext;
import org.genesys.filerepository.service.BytesStorageService;
import org.genesys.filerepository.service.VirusScanner;
import org.genesys.filerepository.service.impl.ClamAVScanner;
import org.genesys.filerepository.service.impl.FilesystemStorageServiceImpl;
import org.genesys.filerepository.service.impl.S3StorageServiceImpl;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.context.annotation.Import;
import org.springframework.core.annotation.Order;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
import org.springframework.security.crypto.password.PasswordEncoder;
@Configuration
@ComponentScan({ "org.genesys.filerepository.service", "org.genesys.catalog.service.impl", "org.genesys.catalog.util", "org.genesys.blocks.security.service.*",
@Import({ FileRepositoryConfig.class })
@ComponentScan({ "org.genesys.catalog.service.impl", "org.genesys.catalog.util", "org.genesys.blocks.security.service.*",
"org.genesys.blocks.auditlog.service.impl", "org.genesys.blocks.security.component" })
@EnableAspectJAutoProxy
public class ApplicationConfig {
@Value("${CLAMD_HOSTNAME}")
private String clamdHost;
@Value("${CLAMD_PORT}")
private String clamdPortStr;
@Bean
public static PropertyPlaceholderConfigurer propertyPlaceholderConfigurer() {
final PropertyPlaceholderConfigurer propertyPlaceholderConfigurer = new PropertyPlaceholderConfigurer();
......@@ -80,44 +67,10 @@ public class ApplicationConfig {
return propertyPlaceholderConfigurer;
}
@Bean
public PasswordEncoder passwordEncoder() {
return new BCryptPasswordEncoder();
}
@Bean
@Order(Integer.MIN_VALUE)
public CurrentApplicationContext currentApplicationContext() {
// This provides static access to current application context
return new CurrentApplicationContext();
}
@Bean
public VirusScanner virusScanner() {
if (StringUtils.isNotBlank(clamdHost)) {
System.err.println("Using clamd at " + clamdHost);
final ClamAVScanner clamav = new ClamAVScanner();
clamav.setClamAvHost(clamdHost);
clamav.setClamAvPort(NumberUtils.toInt(clamdPortStr, 3310));
return clamav;
} else {
System.err.println("clamd host not configured.");
return null;
}
}
@Bean(name = "bytesStorageService")
public BytesStorageService bytesStorageService() {
final File repoDir = new File("data");
final FilesystemStorageServiceImpl storageService = new FilesystemStorageServiceImpl();
storageService.setRepositoryBaseDirectory(repoDir);
return storageService;
}
@Bean(name = "S3Storage")
public BytesStorageService S3StorageService() {
return new S3StorageServiceImpl();
}
}
/*
* Copyright 2017 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys.catalog.server.config;
import java.io.File;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.genesys.filerepository.service.BytesStorageService;
import org.genesys.filerepository.service.ImageGalleryService;
import org.genesys.filerepository.service.RepositoryService;
import org.genesys.filerepository.service.ThumbnailGenerator;
import org.genesys.filerepository.service.VirusScanner;
import org.genesys.filerepository.service.aspect.MetadataInStorageAspect;
import org.genesys.filerepository.service.impl.ClamAVScanner;
import org.genesys.filerepository.service.impl.FilesystemStorageServiceImpl;
import org.genesys.filerepository.service.impl.ImageGalleryServiceImpl;
import org.genesys.filerepository.service.impl.RepositoryServiceImpl;
import org.genesys.filerepository.service.impl.S3StorageServiceImpl;
import org.genesys.filerepository.service.impl.ThumbnailGenerator1;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
/**
* Instantiates and configures the storage mechanism for the FileRepository.
*/
@Configuration
public class FileRepositoryConfig implements InitializingBean {
public static final Logger LOG = LoggerFactory.getLogger(ApplicationConfig.class);
@Value("${CLAMD_HOSTNAME}")
private String clamdHost;
@Value("${CLAMD_PORT}")
private String clamdPortStr;
@Value("${file.repository.dir}")
private String fileRepositoryDir;
@Value("${s3.accessKey}")
private String s3AccessKey;
@Override
public void afterPropertiesSet() throws Exception {
}
/**
* File repository service.
*
* @return the repository service
*/
@Bean
public RepositoryService fileRepositoryService() {
return new RepositoryServiceImpl();
}
@Bean(name = "fileSystemStorage")
@Primary
public BytesStorageService bytesStorageService() {
if (StringUtils.isNotBlank(s3AccessKey)) {
// We have some S3 configuration, let's do that!
LOG.warn("Initializing S3 bytes storage with {}", s3AccessKey);
return new S3StorageServiceImpl();
} else {
// Stick to FS storage
return fileSystemStorage();
}
}
private BytesStorageService fileSystemStorage() {
final FilesystemStorageServiceImpl storageService = new FilesystemStorageServiceImpl();
File repoDir;
if (StringUtils.isNotBlank(fileRepositoryDir)) {
repoDir = new File(fileRepositoryDir);
} else {
repoDir = new File(FileUtils.getTempDirectory(), "repository");
}
LOG.warn("Initializing filesystem bytes storage at base path={}", repoDir.getAbsolutePath());
storageService.setRepositoryBaseDirectory(repoDir);
return storageService;
}
/**
* ImageGallery service.
*
* @return the thing!
*/
@Bean
public ImageGalleryService imageGalleryService() {
return new ImageGalleryServiceImpl();
}
/**
* One Thumbnail generator.
*
* @return the thumbnail generator
*/
@Bean
public ThumbnailGenerator thumbnailGenerator() {
return new ThumbnailGenerator1();
}
/**
* Maintain metadata in .json in storage
*
* @return
*/
@Bean
public MetadataInStorageAspect repositoryMetadataAspects() {
return new MetadataInStorageAspect();
}
/**
* Enable virus scanner if configured
*
* @return
*/
@Bean
public VirusScanner virusScanner() {
if (StringUtils.isNotBlank(clamdHost)) {
System.err.println("Using clamd at " + clamdHost);
final ClamAVScanner clamav = new ClamAVScanner();
clamav.setClamAvHost(clamdHost);
clamav.setClamAvPort(NumberUtils.toInt(clamdPortStr, 3310));
return clamav;
} else {
System.err.println("clamd host not configured.");
return null;
}
}
}
......@@ -25,6 +25,7 @@ import org.springframework.security.config.annotation.method.configuration.Enabl
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
......@@ -36,9 +37,11 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
@Autowired
private UserService userDetailsService;
@Autowired
private PasswordEncoder passwordEncoder;
@Bean
public PasswordEncoder passwordEncoder() {
return new BCryptPasswordEncoder();
}
// @Bean
// public AuthenticationProvider runAsAuthenticationProvider() {
// RunAsImplAuthenticationProvider authProvider = new RunAsImplAuthenticationProvider();
......@@ -50,7 +53,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
protected void configure(final AuthenticationManagerBuilder auth) throws Exception {
auth
//.authenticationProvider(runAsAuthenticationProvider())
.userDetailsService(userDetailsService).passwordEncoder(passwordEncoder);
.userDetailsService(userDetailsService).passwordEncoder(passwordEncoder());
}
@Override
......
......@@ -40,35 +40,37 @@ import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
/**
* Manage dataset files
*
* @author Andrey Lugovskoy.
*/
@RestController
@RequestMapping(RepositoryFileController.API_BASE)
@RequestMapping(DatasetFilesController.API_BASE)
@PreAuthorize("isAuthenticated()")
public class RepositoryFileController {
public class DatasetFilesController {
protected static final String API_BASE = DatasetController.API_BASE + "/{UUID}/files";
private static final Logger LOG = LoggerFactory.getLogger(DatasetController.class);
private static final Logger LOG = LoggerFactory.getLogger(DatasetFilesController.class);
@Autowired
protected DatasetService datasetService;
@PostMapping(value = "/add")
public Dataset addFileToDataset(@RequestParam("file") final MultipartFile inputFile, @PathVariable("UUID") final UUID uuid) throws NotFoundElement,
public Dataset addFileToDataset(@RequestParam("file") final MultipartFile inputFile, @PathVariable("UUID") final UUID datasetUuid) throws NotFoundElement,
InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
LOG.info("Upload file to dataset by uuid {}", uuid);
return datasetService.addDatasetFile(uuid, inputFile);
LOG.info("Upload file to dataset by uuid {}", datasetUuid);
return datasetService.addDatasetFile(datasetService.loadDataset(datasetUuid), inputFile);
}
@DeleteMapping(value = "/delete/{fileUuid}")
public Dataset removeFileOfDataset(@PathVariable("UUID") final UUID datasetUuid, @PathVariable("fileUuid") final UUID fileUuid) throws NotFoundElement,
InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException, NoSuchRepositoryFileException {
return datasetService.removeDatasetFile(datasetUuid, fileUuid);
return datasetService.removeDatasetFile(datasetService.loadDataset(datasetUuid), fileUuid);
}
@GetMapping(value = "/list")
public List<RepositoryFile> getList(@PathVariable("UUID") final UUID datasetUuid) throws NotFoundElement {
return datasetService.listDatasetFiles(datasetUuid);