Commit e7f887a7 authored by Matija Obreza's avatar Matija Obreza
Browse files

Added SHA1 and MD5 sums to RepositoryFile

parent 4b8dd28f
......@@ -177,6 +177,12 @@
<artifactId>thumbnailator</artifactId>
<version>0.4.8</version>
</dependency>
<dependency>
<!-- For SHA1 and MD5 -->
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.10</version>
</dependency>
</dependencies>
<profiles>
......
......@@ -55,9 +55,7 @@ public class RepositoryFile extends BaseEntity implements BaseMetadata {
@Column(name = "path", nullable = false)
private String path;
/**
* The original filename as provided by the end user
*/
/** The original filename as provided by the end user. */
@Column(name = "originalFilename", nullable = false)
private String originalFilename;
......@@ -129,6 +127,16 @@ public class RepositoryFile extends BaseEntity implements BaseMetadata {
@Column
@Temporal(TemporalType.TIMESTAMP)
private Date dateRetrieved;
/** Sha1sum hash of the bytes. */
// TODO Update to nullable = false in next release
@Column(length=40, nullable = true)
private String sha1Sum;
/** MD5 hash of the bytes. */
// TODO Update to nullable = false in next release
@Column(length=32, nullable = true)
private String md5Sum;
/**
* Pre persist.
......@@ -362,7 +370,7 @@ public class RepositoryFile extends BaseEntity implements BaseMetadata {
}
/**
* Gets the content type
* Gets the content type.
*
* @return the content type
*/
......@@ -565,4 +573,40 @@ public class RepositoryFile extends BaseEntity implements BaseMetadata {
public void setDateRetrieved(final Date dateRetrieved) {
this.dateRetrieved = dateRetrieved;
}
/**
* Gets the sha1 sum.
*
* @return the sha1 sum
*/
public String getSha1Sum() {
return sha1Sum;
}
/**
* Sets the sha1 sum.
*
* @param sha1Sum the new sha1 sum
*/
public void setSha1Sum(String sha1Sum) {
this.sha1Sum = sha1Sum;
}
/**
* Gets the md5 sum.
*
* @return the md5 sum
*/
public String getMd5Sum() {
return md5Sum;
}
/**
* Sets the md5 sum.
*
* @param md5Sum the new md5 sum
*/
public void setMd5Sum(String md5Sum) {
this.md5Sum = md5Sum;
}
}
......@@ -36,7 +36,7 @@ public interface BytesStorageService {
void upsert(String path, String filename, byte[] data) throws IOException;
/**
* Removes the.
* Removes the file with specified filename from repository at the specified path
*
* @param path the path
* @param filename the filename
......@@ -60,7 +60,7 @@ public interface BytesStorageService {
* @param path the path
* @param filename the filename
* @return true, if successful
* @throws IOException
* @throws IOException when things go wrong on bytes storage level
*/
boolean exists(String path, String filename) throws IOException;
......
......@@ -45,9 +45,10 @@ public interface RepositoryService {
* @return the repository file
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException when things go wrong on bytes storage level
*/
RepositoryFile addFile(String repositoryPath, String originalFilename, String contentType, byte[] bytes,
RepositoryFile metaData) throws InvalidRepositoryPathException, InvalidRepositoryFileDataException;
RepositoryFile metaData) throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException;
/**
* Add a new image to the file repository.
......@@ -60,9 +61,10 @@ public interface RepositoryService {
* @return the repository image
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException when things go wrong on bytes storage level
*/
RepositoryImage addImage(String repositoryPath, String originalFilename, String contentType, byte[] bytes,
RepositoryImage metaData) throws InvalidRepositoryPathException, InvalidRepositoryFileDataException;
RepositoryImage metaData) throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException;
/**
* Get repository file by its UUID.
......@@ -105,12 +107,12 @@ public interface RepositoryService {
*
* @param fileData the file data
* @param contentType the content type
* @param data the data
* @param bytes the data
* @return the repository file
* @throws NoSuchRepositoryFileException the no such repository file exception
* @throws IOException Signals that an I/O exception has occurred.
*/
RepositoryFile updateBytes(RepositoryFile fileData, String contentType, byte[] data)
RepositoryFile updateBytes(RepositoryFile fileData, String contentType, byte[] bytes)
throws NoSuchRepositoryFileException, IOException;
/**
......@@ -118,12 +120,12 @@ public interface RepositoryService {
*
* @param imageData the image data
* @param contentType the content type
* @param data the data
* @param bytes the data
* @return the repository image
* @throws NoSuchRepositoryFileException the no such repository file exception
* @throws IOException Signals that an I/O exception has occurred.
*/
RepositoryImage updateBytes(RepositoryImage imageData, String contentType, byte[] data)
RepositoryImage updateBytes(RepositoryImage imageData, String contentType, byte[] bytes)
throws NoSuchRepositoryFileException, IOException;
/**
......
......@@ -27,6 +27,7 @@ import java.util.UUID;
import javax.imageio.ImageIO;
import org.apache.commons.codec.digest.DigestUtils;
import org.genesys2.server.filerepository.InvalidRepositoryFileDataException;
import org.genesys2.server.filerepository.InvalidRepositoryPathException;
import org.genesys2.server.filerepository.NoSuchRepositoryFileException;
......@@ -75,7 +76,7 @@ public class RepositoryServiceImpl implements RepositoryService {
@Transactional
public RepositoryFile addFile(final String repositoryPath, final String originalFilename, final String contentType,
final byte[] bytes, final RepositoryFile metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException {
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
if (!PathValidator.isValidPath(repositoryPath)) {
if (LOG.isDebugEnabled()) {
......@@ -87,25 +88,32 @@ public class RepositoryServiceImpl implements RepositoryService {
if (originalFilename == null || contentType == null || bytes == null)
throw new InvalidRepositoryFileDataException();
RepositoryFile file = new RepositoryFile();
RepositoryFile repositoryFile = new RepositoryFile();
if (metaData != null) {
copyMetaData(metaData, file);
copyMetaData(metaData, repositoryFile);
}
file.setOriginalFilename(originalFilename);
file.setContentType(contentType);
file.setPath(repositoryPath);
// Calculate SHA-1 and MD5 sums
repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes));
file = repositoryFilePersistence.save(file);
repositoryFile.setPath(repositoryPath);
repositoryFile.setOriginalFilename(originalFilename);
repositoryFile.setContentType(contentType);
repositoryFile = repositoryFilePersistence.save(repositoryFile);
try {
bytesStorageService.upsert(file.getPath(), file.getFilename(), bytes);
bytesStorageService.upsert(repositoryFile.getPath(), repositoryFile.getFilename(), bytes);
} catch (final IOException e) {
e.printStackTrace();
if (LOG.isDebugEnabled()) {
LOG.debug("Failed to upload bytes", e);
}
throw e;
}
return file;
return repositoryFile;
}
/*
......@@ -118,7 +126,7 @@ public class RepositoryServiceImpl implements RepositoryService {
@Transactional
public RepositoryImage addImage(final String repositoryPath, final String originalFilename,
final String contentType, final byte[] bytes, final RepositoryImage metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException {
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
if (!PathValidator.isValidPath(repositoryPath)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Invalid repository path=" + repositoryPath);
......@@ -129,44 +137,53 @@ public class RepositoryServiceImpl implements RepositoryService {
if (originalFilename == null || contentType == null || bytes == null)
throw new InvalidRepositoryFileDataException();
RepositoryImage image = new RepositoryImage();
RepositoryImage repositoryImage = new RepositoryImage();
if (metaData != null) {
copyMetaData(metaData, image);
copyMetaData(metaData, repositoryImage);
}
image.setPath(repositoryPath);
image.setOriginalFilename(originalFilename);
image.setContentType(contentType);
// Calculate SHA-1 and MD5 sums
repositoryImage.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryImage.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryImage.setPath(repositoryPath);
repositoryImage.setOriginalFilename(originalFilename);
repositoryImage.setContentType(contentType);
try (final InputStream in = new ByteArrayInputStream(bytes)) {
in.mark(0);
final String detectedContentType = URLConnection.guessContentTypeFromStream(in);
if (detectedContentType != null && !detectedContentType.equals(contentType)) {
// overwrite contentType
LOG.warn("Replacing content type; provided={} detected={}", contentType, detectedContentType);
image.setContentType(detectedContentType);
if (LOG.isInfoEnabled()) {
LOG.info("Replacing content type; provided={} detected={}", contentType, detectedContentType);
}
repositoryImage.setContentType(detectedContentType);
}
in.reset();
final BufferedImage imageData = ImageIO.read(in);
if (imageData != null) {
image.setWidth(imageData.getWidth());
image.setHeight(imageData.getHeight());
repositoryImage.setWidth(imageData.getWidth());
repositoryImage.setHeight(imageData.getHeight());
}
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
image = repositoryImagePersistence.save(image);
repositoryImage = repositoryImagePersistence.save(repositoryImage);
try {
bytesStorageService.upsert(image.getPath(), image.getFilename(), bytes);
bytesStorageService.upsert(repositoryImage.getPath(), repositoryImage.getFilename(), bytes);
} catch (final IOException e) {
e.printStackTrace();
if (LOG.isDebugEnabled()) {
LOG.debug("Failed to upload bytes", e);
}
throw e;
}
return image;
return repositoryImage;
}
/*
......@@ -267,15 +284,19 @@ public class RepositoryServiceImpl implements RepositoryService {
*/
@Override
@Transactional
public RepositoryFile updateBytes(final RepositoryFile fileData, final String contentType, final byte[] data)
public RepositoryFile updateBytes(final RepositoryFile repositoryFile, final String contentType, final byte[] bytes)
throws NoSuchRepositoryFileException, IOException {
if (fileData == null)
if (repositoryFile == null)
throw new NoSuchRepositoryFileException();
fileData.setContentType(contentType);
bytesStorageService.upsert(fileData.getPath(), fileData.getFilename(), data);
// Calculate SHA-1 and MD5 sums
repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryFile.setContentType(contentType);
bytesStorageService.upsert(repositoryFile.getPath(), repositoryFile.getFilename(), bytes);
return repositoryFilePersistence.save(fileData);
return repositoryFilePersistence.save(repositoryFile);
}
/*
......@@ -286,24 +307,28 @@ public class RepositoryServiceImpl implements RepositoryService {
*/
@Override
@Transactional
public RepositoryImage updateBytes(final RepositoryImage imageData, final String contentType, final byte[] data)
public RepositoryImage updateBytes(final RepositoryImage repositoryImage, final String contentType, final byte[] bytes)
throws NoSuchRepositoryFileException, IOException {
if (imageData == null)
if (repositoryImage == null)
throw new NoSuchRepositoryFileException();
imageData.setContentType(contentType);
// Calculate SHA-1 and MD5 sums
repositoryImage.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryImage.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryImage.setContentType(contentType);
final InputStream in = new ByteArrayInputStream(data);
final InputStream in = new ByteArrayInputStream(bytes);
final BufferedImage image = ImageIO.read(in);
if (image != null) {
imageData.setWidth(image.getWidth());
imageData.setHeight(image.getHeight());
repositoryImage.setWidth(image.getWidth());
repositoryImage.setHeight(image.getHeight());
}
bytesStorageService.upsert(imageData.getPath(), imageData.getFilename(), data);
bytesStorageService.upsert(repositoryImage.getPath(), repositoryImage.getFilename(), bytes);
return repositoryImagePersistence.save(imageData);
return repositoryImagePersistence.save(repositoryImage);
}
/*
......
......@@ -324,7 +324,6 @@ public class S3StorageServiceImpl implements BytesStorageService {
* @param path the path
* @param filename the filename
* @return true, if successful
* @throws IOException
*/
@Override
public boolean exists(final String path, final String filename) throws IOException {
......
......@@ -20,7 +20,9 @@ import static org.hamcrest.CoreMatchers.*;
import static org.junit.Assert.*;
import java.io.IOException;
import java.util.Random;
import org.apache.commons.codec.digest.DigestUtils;
import org.genesys2.server.filerepository.InvalidRepositoryFileDataException;
import org.genesys2.server.filerepository.InvalidRepositoryPathException;
import org.genesys2.server.filerepository.NoSuchRepositoryFileException;
......@@ -47,6 +49,15 @@ public class FileRepositoryAddTest {
/** The Constant EMPTY_BYTES. */
private static final byte[] EMPTY_BYTES = new byte[0];
/** The Constant PATH. */
private final static String PATH = "/temp/" + System.currentTimeMillis() + "/";
/** The Constant DEFAULT_EXT. */
private static final String DEFAULT_EXT = ".txt";
/** The Constant DEFAULT_FILENAME. */
private static final String DEFAULT_FILENAME = "test" + DEFAULT_EXT;
/** The file repo service. */
@Autowired
private RepositoryService fileRepoService;
......@@ -56,17 +67,17 @@ public class FileRepositoryAddTest {
*
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException
*/
@Test
public void addFile() throws InvalidRepositoryPathException, InvalidRepositoryFileDataException {
final String path = "/test1/test2/";
public void addFile() throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
final String contentType = "text/plain;charset=UTF-8";
final String extension = ".txt";
final String originalFilename = "requirements" + extension;
final RepositoryFile repoFile = fileRepoService.addFile(path, originalFilename, contentType, SOME_BYTES, null);
final RepositoryFile repoFile = fileRepoService.addFile(PATH, originalFilename, contentType, SOME_BYTES, null);
FileRepositoryTestUtil.checkFile(repoFile, path, originalFilename, extension, contentType);
FileRepositoryTestUtil.checkFile(repoFile, PATH, originalFilename, extension, contentType);
}
/**
......@@ -74,9 +85,10 @@ public class FileRepositoryAddTest {
*
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException
*/
@Test
public void addFileWithMetadata() throws InvalidRepositoryPathException, InvalidRepositoryFileDataException {
public void addFileWithMetadata() throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
final String path = "/test1/test2/";
final String contentType = "text/plain;charset=UTF-8";
final String extension = ".txt";
......@@ -129,10 +141,11 @@ public class FileRepositoryAddTest {
*
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryFileDataException.class)
public void invalidMissingOriginalFilename()
throws InvalidRepositoryFileDataException, InvalidRepositoryPathException {
throws InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
fileRepoService.addFile("/valid/path/", null, "contentType", FileRepositoryAddTest.EMPTY_BYTES, null);
}
......@@ -141,9 +154,10 @@ public class FileRepositoryAddTest {
*
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryFileDataException.class)
public void invalidMissingContentType() throws InvalidRepositoryFileDataException, InvalidRepositoryPathException {
public void invalidMissingContentType() throws InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
fileRepoService.addFile("/valid/path/", "orignalFilename.txt", null, FileRepositoryAddTest.EMPTY_BYTES, null);
}
......@@ -152,9 +166,10 @@ public class FileRepositoryAddTest {
*
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryFileDataException.class)
public void invalidMissingBytes() throws InvalidRepositoryFileDataException, InvalidRepositoryPathException {
public void invalidMissingBytes() throws InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
fileRepoService.addFile("/valid/path/", "orignalFilename.txt", "contentType", null, null);
}
......@@ -164,10 +179,11 @@ public class FileRepositoryAddTest {
* @throws NoSuchRepositoryFileException the no such repository file exception
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryPathException.class)
public void invalidAddNullPath()
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException {
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
fileRepoService.addFile(null, "originalFilename.txt", "contentType", FileRepositoryAddTest.EMPTY_BYTES, null);
}
......@@ -177,10 +193,11 @@ public class FileRepositoryAddTest {
* @throws NoSuchRepositoryFileException the no such repository file exception
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryPathException.class)
public void invalidAddBlankPath()
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException {
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
fileRepoService.addFile(" ", "originalFilename.txt", "contentType", FileRepositoryAddTest.EMPTY_BYTES, null);
}
......@@ -190,10 +207,11 @@ public class FileRepositoryAddTest {
* @throws NoSuchRepositoryFileException the no such repository file exception
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryPathException.class)
public void invalidAddDoubleSlashPath()
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException {
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
fileRepoService.addFile("/double//slash/", "originalFilename.txt", "contentType",
FileRepositoryAddTest.EMPTY_BYTES, null);
}
......@@ -203,10 +221,11 @@ public class FileRepositoryAddTest {
*
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryPathException.class)
public void invalidAddMissingStartSlash()
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException {
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
fileRepoService.addFile("invalidpath/here/", "originalFilename.txt", "contentType",
FileRepositoryAddTest.EMPTY_BYTES, null);
}
......@@ -217,10 +236,11 @@ public class FileRepositoryAddTest {
* @throws NoSuchRepositoryFileException the no such repository file exception
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryPathException.class)
public void invalidAddMissingEndSlash()
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException {
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
fileRepoService.addFile("/invalidpath/here", "originalFilename.txt", "contentType",
FileRepositoryAddTest.EMPTY_BYTES, null);
}
......@@ -255,10 +275,11 @@ public class FileRepositoryAddTest {
* @throws NoSuchRepositoryFileException the no such repository file exception
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryPathException.class)
public void invalidSpaceNextToSlashInPath1()
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException {
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
fileRepoService.addFile("/invalidpath/here /", "originalFilename.txt", "contentType",
FileRepositoryAddTest.EMPTY_BYTES, null);
}
......@@ -269,11 +290,71 @@ public class FileRepositoryAddTest {
* @throws NoSuchRepositoryFileException the no such repository file exception
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws InvalidRepositoryFileDataException the invalid repository file data exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryPathException.class)
public void invalidSpaceNextToSlashInPath2()
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException {
throws NoSuchRepositoryFileException, InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
fileRepoService.addFile("/invalidpath/ orhere/", "originalFilename.txt", "contentType",
FileRepositoryAddTest.EMPTY_BYTES, null);
}
/**
* Check that SHA1 and MD5 sums are generated on add
*
* @throws InvalidRepositoryPathException
* @throws InvalidRepositoryFileDataException
* @throws IOException
* @throws NoSuchRepositoryFileException
*/
@Test
public void testSha1AndMd5SumsOnAdd() throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, NoSuchRepositoryFileException, IOException {
final String contentType = "text/plain;charset=UTF-8";
byte[] randomBytes=new byte[100];
Random rnd=new Random(System.currentTimeMillis());
rnd.nextBytes(randomBytes);
final RepositoryFile repositoryFile = fileRepoService.addFile(PATH, DEFAULT_FILENAME, contentType, randomBytes, null);
FileRepositoryTestUtil.checkFile(repositoryFile, PATH, DEFAULT_FILENAME, DEFAULT_EXT, contentType);
String sha1hex = DigestUtils.sha1Hex(randomBytes);
String md5hex = DigestUtils.md5Hex(randomBytes);
assertThat("sha1 does must match", repositoryFile.getSha1Sum(), equalTo(sha1hex));
assertThat("md5 does must match", repositoryFile.getMd5Sum(), equalTo(md5hex));
fileRepoService.removeFile(repositoryFile);
}
/**