Commit f2813031 authored by Matija Obreza's avatar Matija Obreza
Browse files

Maintain metadata as JSON in bytes storage next to file itself

parent 20775b20
......@@ -182,5 +182,10 @@
<artifactId>clamav-client</artifactId>
<version>1.0.4</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.8.8</version>
</dependency>
</dependencies>
</project>
/*
* Copyright 2017 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.filerepository;
import org.springframework.data.jpa.repository.JpaRepository;
public interface RepositoryPersistence<T> extends JpaRepository<T, Long> {
}
......@@ -127,7 +127,8 @@ public class RepositoryFile extends AuditedVersionedModelWithoutId implements En
private String originalUrl;
/**
* For resources retrieved from {@link #originalUrl} we maintain the date of retrieval.
* For resources retrieved from {@link #originalUrl} we maintain the date of
* retrieval.
*/
@Column
@Temporal(TemporalType.TIMESTAMP)
......@@ -167,7 +168,8 @@ public class RepositoryFile extends AuditedVersionedModelWithoutId implements En
}
/**
* Get the relative URL to the resource. As such, it cannot serve as an {@link #getIdentifier()} property.
* Get the relative URL to the resource. As such, it cannot serve as an
* {@link #getIdentifier()} property.
*
* @return relative URL to the file resource
*/
......@@ -196,6 +198,21 @@ public class RepositoryFile extends AuditedVersionedModelWithoutId implements En
return sb.toString();
}
/**
* Gets the filename for metadata as saved in {@link BytesStorageService}.
*
* @return the filename
*/
// not persisted
@Transient
public String getMetadataFilename() {
if (uuid == null) {
return null;
}
return new StringBuffer().append(uuid.toString()).append(".json").toString();
}
/**
* Get the path of the file used by {@link BytesStorageService}
*
......@@ -210,7 +227,8 @@ public class RepositoryFile extends AuditedVersionedModelWithoutId implements En
}
/**
* Get the full path to the file as used by {@link BytesStorageService}. This is the concatenation of {@link #getStoragePath()} and {@link #getFilename()}.
* Get the full path to the file as used by {@link BytesStorageService}. This is
* the concatenation of {@link #getStoragePath()} and {@link #getFilename()}.
*
* @return
*/
......@@ -437,7 +455,8 @@ public class RepositoryFile extends AuditedVersionedModelWithoutId implements En
/*
* (non-Javadoc)
* @see org.genesys.filerepository.metadata.BaseMetadata# getBibliographicCitation()
* @see org.genesys.filerepository.metadata.BaseMetadata#
* getBibliographicCitation()
*/
@Override
public String getBibliographicCitation() {
......
......@@ -19,8 +19,8 @@ package org.genesys.filerepository.persistence;
import java.util.List;
import java.util.UUID;
import org.genesys.filerepository.RepositoryPersistence;
import org.genesys.filerepository.model.RepositoryDocument;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
// TODO: Auto-generated Javadoc
......@@ -28,7 +28,7 @@ import org.springframework.stereotype.Repository;
* The Interface RepositoryDocumentPersistence.
*/
@Repository
public interface RepositoryDocumentPersistence extends JpaRepository<RepositoryDocument, Long> {
public interface RepositoryDocumentPersistence extends RepositoryPersistence<RepositoryDocument> {
/**
* Find by uuid.
......
......@@ -20,9 +20,9 @@ import java.util.Collection;
import java.util.List;
import java.util.UUID;
import org.genesys.filerepository.RepositoryPersistence;
import org.genesys.filerepository.model.RepositoryFile;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
......@@ -31,7 +31,7 @@ import org.springframework.stereotype.Repository;
* The Interface RepositoryFilePersistence.
*/
@Repository
public interface RepositoryFilePersistence extends JpaRepository<RepositoryFile, Long> {
public interface RepositoryFilePersistence extends RepositoryPersistence<RepositoryFile> {
/**
* Find by uuid.
......
......@@ -19,8 +19,8 @@ package org.genesys.filerepository.persistence;
import java.util.List;
import java.util.UUID;
import org.genesys.filerepository.RepositoryPersistence;
import org.genesys.filerepository.model.RepositoryImage;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
// TODO: Auto-generated Javadoc
......@@ -28,7 +28,7 @@ import org.springframework.stereotype.Repository;
* The Interface RepositoryImagePersistence.
*/
@Repository
public interface RepositoryImagePersistence extends JpaRepository<RepositoryImage, Long> {
public interface RepositoryImagePersistence extends RepositoryPersistence<RepositoryImage> {
/**
* Find by uuid.
......
......@@ -121,7 +121,7 @@ public interface RepositoryService {
* @return the updated RepositoryFile
* @throws NoSuchRepositoryFileException when file is not available in the repository
*/
RepositoryFile updateMetadata(UUID uuid, RepositoryFile fileData) throws NoSuchRepositoryFileException;
<T extends RepositoryFile> T updateMetadata(UUID uuid, T fileData) throws NoSuchRepositoryFileException;
/**
* Update file bytes.
......
/*
* Copyright 2017 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.filerepository.service.aspect;
import java.io.IOException;
import java.nio.charset.Charset;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
import org.genesys.filerepository.model.RepositoryFile;
import org.genesys.filerepository.service.BytesStorageService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Make sure that repository metadata is persisted as .json next to the data
* bytes in {@link BytesStorageService}. Any change to the repository records
* triggers a regeneration and rewrite of the .json file.
*
* On delete, the .json file is removed from {@link BytesStorageService}.
*
* @author Matija Obreza
*/
@Aspect
public class MetadataInStorageAspect {
/** The Constant LOG. */
protected Logger LOG = LoggerFactory.getLogger(getClass());
private static final Charset UTF8 = Charset.forName("UTF8");
@Autowired
private BytesStorageService bytesStorageService;
private final ObjectWriter objectWriter;
{
LOG.warn("Instantiated " + getClass().getName());
objectWriter = new ObjectMapper().writerWithDefaultPrettyPrinter();
}
@AfterReturning(value = "execution(* org.genesys.filerepository.RepositoryPersistence.save(*))", returning = "repositoryFiles")
public Object afterRepositoryImageSaveIterable(final JoinPoint joinPoint, final Iterable<RepositoryFile> repositoryFiles) {
LOG.debug("Many files were saved: {}", repositoryFiles);
if (repositoryFiles != null) {
repositoryFiles.forEach(rf -> {
try {
updateMetadata((RepositoryFile) rf);
} catch (IOException e) {
LOG.error(e.getMessage());
}
});
}
return repositoryFiles;
}
@AfterReturning(value = "execution(* org.genesys.filerepository.RepositoryPersistence.save(*))", returning = "repositoryFile")
public Object afterRepositoryImageSave(final JoinPoint joinPoint, RepositoryFile repositoryFile) {
if (LOG.isTraceEnabled()) {
LOG.trace("1 file was saved: {}", repositoryFile);
}
try {
updateMetadata(repositoryFile);
} catch (IOException e) {
LOG.error(e.getMessage());
}
return repositoryFile;
}
@AfterReturning(value = "execution(* org.genesys.filerepository.RepositoryPersistence.delete(*)) && args(repositoryFiles)")
public Object afterRepositoryFilesDelete(final JoinPoint joinPoint, final Iterable<RepositoryFile> repositoryFiles) {
if (repositoryFiles != null) {
repositoryFiles.forEach(rf -> {
try {
removeMetadata((RepositoryFile) rf);
} catch (IOException e) {
LOG.error(e.getMessage());
}
});
}
return repositoryFiles;
}
@AfterReturning(value = "execution(* org.genesys.filerepository.RepositoryPersistence.delete(*)) && args(repositoryFile)")
public Object afterRepositoryImageDelete(final JoinPoint joinPoint, final RepositoryFile repositoryFile) {
try {
removeMetadata((RepositoryFile) repositoryFile);
} catch (IOException e) {
LOG.error(e.getMessage());
}
return repositoryFile;
}
private void updateMetadata(RepositoryFile repositoryFile) throws IOException {
if (repositoryFile == null) {
return;
}
LOG.trace("File was updated path={} originalFilename={}. Writing metadata.json", repositoryFile.getPath(), repositoryFile.getOriginalFilename());
try {
String metadataJson = objectWriter.writeValueAsString(repositoryFile);
try {
bytesStorageService.upsert(repositoryFile.getStoragePath(), repositoryFile.getMetadataFilename(), metadataJson.getBytes(UTF8));
} catch (final IOException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Failed to upload metadata bytes", e);
}
throw e;
}
} catch (JsonProcessingException e) {
LOG.error("Could not serialize repositoryFile to JSON: " + e.getMessage(), e);
}
}
private void removeMetadata(RepositoryFile repositoryFile) throws IOException {
if (repositoryFile == null) {
return;
}
LOG.trace("File was deleted path={} originalFilename={}. Removing metadata.json", repositoryFile.getPath(), repositoryFile.getOriginalFilename());
try {
bytesStorageService.remove(repositoryFile.getStoragePath(), repositoryFile.getMetadataFilename());
} catch (final IOException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Failed to delete metadata bytes", e);
}
throw e;
}
}
}
\ No newline at end of file
......@@ -88,11 +88,12 @@ public class FilesystemStorageServiceImpl implements BytesStorageService {
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.BytesStorageService#upsert( java.lang.String, java.lang.String, byte[])
* @see org.genesys.filerepository.service.BytesStorageService#upsert(
* java.lang.String, java.lang.String, byte[])
*/
@Override
public void upsert(final String path, final String filename, final byte[] data) throws FileNotFoundException, IOException {
LOG.trace("Trying to upsert path={} filename={}", path, filename);
final File destinationDir = new File(repoDir, path);
if (!destinationDir.getCanonicalPath().startsWith(repoDir.getCanonicalPath())) {
......@@ -115,8 +116,8 @@ public class FilesystemStorageServiceImpl implements BytesStorageService {
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.BytesStorageService#remove( java.lang.String, java.lang.String)
* @see org.genesys.filerepository.service.BytesStorageService#remove(
* java.lang.String, java.lang.String)
*/
@Override
public void remove(final String path, final String filename) throws IOException {
......@@ -146,12 +147,12 @@ public class FilesystemStorageServiceImpl implements BytesStorageService {
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.BytesStorageService#get(java. lang.String, java.lang.String)
* @see org.genesys.filerepository.service.BytesStorageService#get(java.
* lang.String, java.lang.String)
*/
@Override
public byte[] get(final String path, final String filename) throws IOException {
LOG.trace("Retrieveing bytes of {} {}", path, filename);
LOG.trace("Retrieving bytes of {} {}", path, filename);
final File destinationDir = new File(repoDir, path);
final File destinationFile = new File(destinationDir, filename);
......@@ -173,8 +174,8 @@ public class FilesystemStorageServiceImpl implements BytesStorageService {
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.BytesStorageService#exists(java.lang.String, java.lang.String)
* @see org.genesys.filerepository.service.BytesStorageService#exists(java.lang.
* String, java.lang.String)
*/
@Override
public boolean exists(final String path, final String filename) {
......@@ -186,8 +187,9 @@ public class FilesystemStorageServiceImpl implements BytesStorageService {
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.BytesStorageService#listFiles(java.lang.String)
* @see
* org.genesys.filerepository.service.BytesStorageService#listFiles(java.lang.
* String)
*/
@Override
public List<String> listFiles(final String path) {
......@@ -197,7 +199,7 @@ public class FilesystemStorageServiceImpl implements BytesStorageService {
final File destinationDir = new File(repoDir, path);
if (! destinationDir.exists() || ! destinationDir.isDirectory()) {
if (!destinationDir.exists() || !destinationDir.isDirectory()) {
LOG.info("Returning empty files list for nonexistent dir={}", destinationDir.getAbsolutePath());
return Collections.emptyList();
}
......
......@@ -286,21 +286,22 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
* (non-Javadoc)
* @see org.genesys.filerepository.service.RepositoryService#updateFile( org.genesys.filerepository.model .RepositoryFile)
*/
@SuppressWarnings("unchecked")
@Override
@Transactional
public RepositoryFile updateMetadata(final UUID uuid, final RepositoryFile fileData) throws NoSuchRepositoryFileException {
public <T extends RepositoryFile> T updateMetadata(final UUID uuid, final T fileData) throws NoSuchRepositoryFileException {
RepositoryFile repositoryFile = repositoryFilePersistence.findByUuid(uuid);
if (repositoryFile == null) {
throw new NoSuchRepositoryFileException();
}
if (fileData instanceof RepositoryImage) {
return updateImageMetadata(uuid, (RepositoryImage) fileData);
return (T) updateImageMetadata(uuid, (RepositoryImage) fileData);
}
repositoryFile.apply(fileData);
repositoryFile = repositoryFilePersistence.save(repositoryFile);
return repositoryFile;
return (T) repositoryFile;
}
/*
......
......@@ -24,6 +24,7 @@ import org.genesys.filerepository.service.RepositoryService;
import org.genesys.filerepository.service.ThumbnailGenerator;
import org.genesys.filerepository.service.aspect.AbstractImageGalleryAspects;
import org.genesys.filerepository.service.aspect.ImageGalleryAspectsImpl;
import org.genesys.filerepository.service.aspect.MetadataInStorageAspect;
import org.genesys.filerepository.service.impl.FilesystemStorageServiceImpl;
import org.genesys.filerepository.service.impl.ImageGalleryServiceImpl;
import org.genesys.filerepository.service.impl.RepositoryServiceImpl;
......@@ -126,6 +127,11 @@ public class ServiceBeanConfig {
public AbstractImageGalleryAspects imageGalleryAspects() {
return new ImageGalleryAspectsImpl();
}
@Bean
public MetadataInStorageAspect metadataInStorageAspect() {
return new MetadataInStorageAspect();
}
/**
* One Thumbnail generator.
......
/*
* Copyright 2016 Global Crop Diversity Trust, www.croptrust.org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.filerepository.service;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
import java.io.IOException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
import org.genesys.filerepository.config.DatabaseConfig;
import org.genesys.filerepository.config.ServiceBeanConfig;
import org.genesys.filerepository.model.RepositoryImage;
import org.junit.After;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* Repository metadata tests
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = { ServiceBeanConfig.class, DatabaseConfig.class })
public class MetadataTest {
/** The timestamp. */
private final long timestamp = System.currentTimeMillis();
/** The initial path. */
private final String initialPath = "/images/" + timestamp;
private final ObjectMapper objectMapper;
/** The repository service. */
@Autowired
private RepositoryService repositoryService;
@Autowired
private BytesStorageService bytesStorageService;
{
objectMapper = new ObjectMapper();
objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
}
/**
* After test.
*/
@After
public void afterTest() {
}
/**
* Load missing gallery.
*
* @throws IOException
* @throws InvalidRepositoryFileDataException
* @throws InvalidRepositoryPathException
* @throws NoSuchRepositoryFileException
*/
@Test
public void createImageAndMetadata() throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException, NoSuchRepositoryFileException {
final TestImage image1 = new TestImage("10x10.png", "image/png");
RepositoryImage repoImage1 = repositoryService.addImage(initialPath, image1.getOriginalFilename(), image1.getContentType(), image1.getImageBytes(), null);
assertThat(repoImage1.getUuid(), not(nullValue()));
byte[] metadata = bytesStorageService.get(repoImage1.getStoragePath(), repoImage1.getMetadataFilename());
assertThat("Metadata .json not found", metadata, not(nullValue()));
RepositoryImage meta = objectMapper.readValue(metadata, RepositoryImage.class);
assertThat(meta.getOriginalFilename(), is(repoImage1.getOriginalFilename()));
assertThat(meta.getContentType(), is("image/png"));
assertThat(meta.getWidth(), is(repoImage1.getWidth()));
assertThat(meta.getHeight(), is(repoImage1.getHeight()));
assertThat(meta.getWidth(), is(7));
assertThat(meta.getHeight(), is(7));
repoImage1.setOriginalFilename("test.png");
repoImage1 = (RepositoryImage) repositoryService.updateMetadata(repoImage1.getUuid(), repoImage1);
metadata = bytesStorageService.get(repoImage1.getStoragePath(), repoImage1.getMetadataFilename());
meta = objectMapper.readValue(metadata, RepositoryImage.class);
assertThat(meta.getOriginalFilename(), is(repoImage1.getOriginalFilename()));
repositoryService.removeFile(repoImage1);
metadata = bytesStorageService.get(repoImage1.getStoragePath(), repoImage1.getMetadataFilename());
assertThat(metadata, is(nullValue()));
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment