Commit 4b689a4b authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch '8-ftp-repositoryfilesystemview' into 'master'

Resolve "(ftp) RepositoryFilesystemView"

Closes #8

See merge request !6
parents d24b7510 929344b6
......@@ -6,3 +6,4 @@
target
doc
data
ftpserver.jks
......@@ -28,6 +28,7 @@
<properties>
<commons.io.version>2.4</commons.io.version>
<tika.version>1.14</tika.version>
</properties>
<build>
......@@ -59,6 +60,18 @@
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
......@@ -159,5 +172,10 @@
<version>${querydsl.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>${tika.version}</version>
</dependency>
</dependencies>
</project>
......@@ -22,6 +22,13 @@ package org.genesys.filerepository;
*/
public class InvalidRepositoryFileDataException extends FileRepositoryException {
public InvalidRepositoryFileDataException() {
}
public InvalidRepositoryFileDataException(String message) {
super(message);
}
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
......
......@@ -16,12 +16,18 @@
package org.genesys.filerepository;
// TODO: Auto-generated Javadoc
/**
* The Class NoSuchRepositoryFileException.
*/
public class NoSuchRepositoryFileException extends FileRepositoryException {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
public NoSuchRepositoryFileException() {
}
public NoSuchRepositoryFileException(String message) {
super(message);
}
}
......@@ -29,6 +29,7 @@ import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.persistence.Transient;
import javax.persistence.UniqueConstraint;
import org.genesys.blocks.model.AuditedVersionedModel;
import org.genesys.blocks.model.Copyable;
......@@ -39,7 +40,11 @@ import org.genesys.filerepository.metadata.BaseMetadata;
* The Class RepositoryFile.
*/
@Entity
@Table(name = "repositoryfile", indexes = { @Index(unique = false, columnList = "path", name = "IX_repoFile_path") })
@Table(name = "repositoryfile",
// indexes
indexes = { @Index(unique = false, columnList = "path", name = "IX_repoFile_path") }
// unique
, uniqueConstraints = { @UniqueConstraint(columnNames = { "path", "originalFilename" }) })
@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS)
public class RepositoryFile extends AuditedVersionedModel implements BaseMetadata, Copyable<RepositoryFile> {
......@@ -119,15 +124,17 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat
private Date dateRetrieved;
/** Sha1sum hash of the bytes. */
// TODO Update to nullable = false in next release
@Column(length = 40, nullable = true)
@Column(length = 40, nullable = false)
private String sha1Sum;
/** MD5 hash of the bytes. */
// TODO Update to nullable = false in next release
@Column(length = 32, nullable = true)
@Column(length = 32, nullable = false)
private String md5Sum;
/** Byte length */
@Column(nullable = false)
private int size;
/**
* Pre persist.
*/
......@@ -568,6 +575,20 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat
this.md5Sum = md5Sum;
}
/**
* @return the size
*/
public final int getSize() {
return size;
}
/**
* @param size the size to set
*/
public final void setSize(int size) {
this.size = size;
}
@Override
public RepositoryFile apply(RepositoryFile source) {
......@@ -590,6 +611,7 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat
this.subject = source.subject;
this.title = source.title;
this.uuid = source.uuid;
this.size = source.size;
return this;
}
......
......@@ -58,6 +58,15 @@ public interface RepositoryFilePersistence extends JpaRepository<RepositoryFile,
@Query("select distinct(rf.path) from RepositoryFile rf where rf.path like ?1%")
List<String> listDistinctPaths(String prefix, Pageable pageable);
/**
* List all distinct paths (FTP)
*
* @param prefix the prefix
* @return the list of paths
*/
@Query("select distinct(rf.path) from RepositoryFile rf where rf.path like ?1%")
List<String> listDistinctPaths(String prefix);
/**
* Find files with missing hashes
*
......@@ -65,4 +74,10 @@ public interface RepositoryFilePersistence extends JpaRepository<RepositoryFile,
*/
@Query("select rf from RepositoryFile rf where rf.sha1Sum is null or rf.md5Sum is null")
List<RepositoryFile> findByMissingHashSums();
/**
* Find repository file by path and originalFilename
*/
RepositoryFile findByPathAndOriginalFilename(String path, String originalFilename);
}
......@@ -75,6 +75,16 @@ public interface RepositoryService {
*/
RepositoryFile getFile(UUID fileUuid) throws NoSuchRepositoryFileException;
/**
* Get repository file by its path and filename
*
* @param path
* @param filename
* @return
* @throws NoSuchRepositoryFileException
*/
RepositoryFile getFile(String path, String filename) throws NoSuchRepositoryFileException;
/**
* Get repository file bytes by its path and filename.
*
......@@ -174,6 +184,14 @@ public interface RepositoryService {
*/
List<String> listPaths(String prefix, Pageable pageRequest);
/**
* List all paths
*
* @param prefix
* @return
*/
List<String> listPaths(String prefix);
/**
* Update image metadata. The update is based on the record UUID.
*
......
......@@ -17,10 +17,10 @@
package org.genesys.filerepository.service.impl;
import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
......@@ -28,6 +28,7 @@ import java.util.UUID;
import javax.imageio.ImageIO;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.tika.Tika;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
......@@ -95,16 +96,18 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.RepositoryService#addFile(java .lang.String, java.lang.String, java.lang.String, byte[],
* org.genesys.filerepository.model.RepositoryFile)
* @see org.genesys.filerepository.service.RepositoryService#addFile(java .lang.String, java.lang.String,
* java.lang.String, byte[], org.genesys.filerepository.model.RepositoryFile)
*/
@Override
@Transactional
public RepositoryFile addFile(final String repositoryPath, final String originalFilename, final String contentType, final byte[] bytes, final RepositoryFile metaData)
public RepositoryFile addFile(final String repositoryPath, final String originalFilename, String contentType, final byte[] bytes, final RepositoryFile metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
PathValidator.checkValidPath(repositoryPath);
contentType = updateContentTypeIfNecessary(contentType, bytes);
if (originalFilename == null || contentType == null || bytes == null) {
throw new InvalidRepositoryFileDataException();
}
......@@ -118,11 +121,16 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
// Calculate SHA-1 and MD5 sums
repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryFile.setSize(bytes.length);
repositoryFile.setPath(repositoryPath);
repositoryFile.setOriginalFilename(originalFilename);
repositoryFile.setContentType(contentType);
if (contentType == null || contentType.length() == 0) {
throw new InvalidRepositoryFileDataException("Content type not privided and could not be detected");
}
repositoryFile = repositoryFilePersistence.save(repositoryFile);
try {
......@@ -140,16 +148,18 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.RepositoryService#addImage( java.lang.String, java.lang.String, java.lang.String, byte[],
* org.genesys.filerepository.model.RepositoryImage)
* @see org.genesys.filerepository.service.RepositoryService#addImage( java.lang.String, java.lang.String,
* java.lang.String, byte[], org.genesys.filerepository.model.RepositoryImage)
*/
@Override
@Transactional
public RepositoryImage addImage(final String repositoryPath, final String originalFilename, final String contentType, final byte[] bytes, final RepositoryImage metaData)
public RepositoryImage addImage(final String repositoryPath, final String originalFilename, String contentType, final byte[] bytes, final RepositoryImage metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
PathValidator.checkValidPath(repositoryPath);
contentType = updateContentTypeIfNecessary(contentType, bytes);
if (originalFilename == null || contentType == null || bytes == null) {
throw new InvalidRepositoryFileDataException();
}
......@@ -163,31 +173,13 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
// Calculate SHA-1 and MD5 sums
repositoryImage.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryImage.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryImage.setSize(bytes.length);
repositoryImage.setPath(repositoryPath);
repositoryImage.setOriginalFilename(originalFilename);
repositoryImage.setContentType(contentType);
try (final InputStream in = new ByteArrayInputStream(bytes)) {
in.mark(0);
final String detectedContentType = URLConnection.guessContentTypeFromStream(in);
if (detectedContentType != null && !detectedContentType.equals(contentType)) {
// overwrite contentType
if (LOG.isInfoEnabled()) {
LOG.info("Replacing content type; provided={} detected={}", contentType, detectedContentType);
}
repositoryImage.setContentType(detectedContentType);
}
in.reset();
final BufferedImage imageData = ImageIO.read(in);
if (imageData != null) {
repositoryImage.setWidth(imageData.getWidth());
repositoryImage.setHeight(imageData.getHeight());
}
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
fillImageProperties(repositoryImage, bytes);
repositoryImage = repositoryImagePersistence.save(repositoryImage);
......@@ -203,6 +195,24 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
return repositoryImage;
}
/**
* Load image to get pixel width and height
*
* @param repositoryImage
* @param bytes
*/
private void fillImageProperties(RepositoryImage repositoryImage, byte[] bytes) {
try (final InputStream in = new BufferedInputStream(new ByteArrayInputStream(bytes))) {
final BufferedImage imageData = ImageIO.read(in);
if (imageData != null) {
repositoryImage.setWidth(imageData.getWidth());
repositoryImage.setHeight(imageData.getHeight());
}
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
}
/*
* (non-Javadoc)
*
......@@ -223,6 +233,15 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
return file;
}
@Override
public RepositoryFile getFile(final String path, final String originalFilename) throws NoSuchRepositoryFileException {
RepositoryFile repositoryFile = repositoryFilePersistence.findByPathAndOriginalFilename(path, originalFilename);
if (repositoryFile == null) {
throw new NoSuchRepositoryFileException("No file at path=" + path + " originalFilename=" + originalFilename);
}
return repositoryFile;
}
/*
* (non-Javadoc)
*
......@@ -262,7 +281,8 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.RepositoryService#updateFile( org.genesys.filerepository.model .RepositoryFile)
* @see org.genesys.filerepository.service.RepositoryService#updateFile( org.genesys.filerepository.model
* .RepositoryFile)
*/
@Override
@Transactional
......@@ -284,7 +304,8 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.RepositoryService#updateMetadata(org.genesys.filerepository. model.RepositoryImage)
* @see org.genesys.filerepository.service.RepositoryService#updateMetadata(org.genesys.filerepository.
* model.RepositoryImage)
*/
@Override
@Transactional
......@@ -303,11 +324,12 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.RepositoryService#updateBytes( org.genesys.filerepository.model .RepositoryFile, java.lang.String, byte[])
* @see org.genesys.filerepository.service.RepositoryService#updateBytes( org.genesys.filerepository.model
* .RepositoryFile, java.lang.String, byte[])
*/
@Override
@Transactional
public RepositoryFile updateBytes(final RepositoryFile repositoryFile, final String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException {
public RepositoryFile updateBytes(final RepositoryFile repositoryFile, String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException {
if (repositoryFile == null) {
throw new NoSuchRepositoryFileException();
}
......@@ -315,9 +337,14 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
return updateImageBytes((RepositoryImage) repositoryFile, contentType, bytes);
}
contentType = updateContentTypeIfNecessary(contentType, bytes);
// Calculate SHA-1 and MD5 sums
LOG.debug("updateByes length={}", bytes.length);
repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryFile.setSize(bytes.length);
LOG.debug("updateByes length={} repoFile.size={}", bytes.length, repositoryFile.getSize());
repositoryFile.setContentType(contentType);
bytesStorageService.upsert(repositoryFile.getPath(), repositoryFile.getFilename(), bytes);
......@@ -328,38 +355,48 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.RepositoryService#updateBytes( org.genesys.filerepository.model .RepositoryImage, java.lang.String, byte[])
* @see org.genesys.filerepository.service.RepositoryService#updateBytes( org.genesys.filerepository.model
* .RepositoryImage, java.lang.String, byte[])
*/
@Override
@Transactional
public RepositoryImage updateImageBytes(final RepositoryImage repositoryImage, final String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException {
public RepositoryImage updateImageBytes(final RepositoryImage repositoryImage, String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException {
if (repositoryImage == null) {
throw new NoSuchRepositoryFileException();
}
contentType = updateContentTypeIfNecessary(contentType, bytes);
// Calculate SHA-1 and MD5 sums
repositoryImage.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryImage.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryImage.setSize(bytes.length);
repositoryImage.setContentType(contentType);
final InputStream in = new ByteArrayInputStream(bytes);
final BufferedImage image = ImageIO.read(in);
if (image != null) {
repositoryImage.setWidth(image.getWidth());
repositoryImage.setHeight(image.getHeight());
}
fillImageProperties(repositoryImage, bytes);
bytesStorageService.upsert(repositoryImage.getPath(), repositoryImage.getFilename(), bytes);
return repositoryImagePersistence.save(repositoryImage);
}
private String updateContentTypeIfNecessary(String contentType, byte[] bytes) {
if (bytes == null) {
return contentType;
}
if (contentType == null) {
return new Tika().detect(bytes);
} else {
String detectedContentType = new Tika().detect(bytes);
LOG.info("Content-Type provided={} detected={}", contentType, detectedContentType);
return contentType;
}
}
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.RepositoryService#removeFile( org.genesys.filerepository.model .RepositoryFile)
* @see org.genesys.filerepository.service.RepositoryService#removeFile( org.genesys.filerepository.model
* .RepositoryFile)
*/
@Override
@Transactional
......@@ -392,7 +429,8 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/*
* (non-Javadoc)
*
* @see org.genesys.filerepository.service.RepositoryService#moveFile(org .genesys2.server.filerepository.model .RepositoryFile, java.lang.String)
* @see org.genesys.filerepository.service.RepositoryService#moveFile(org .genesys2.server.filerepository.model
* .RepositoryFile, java.lang.String)
*/
@Override
@Transactional
......@@ -462,4 +500,11 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
return paths;
}
@Override
public List<String> listPaths(String prefix) {
final List<String> paths = repositoryFilePersistence.listDistinctPaths(prefix);
paths.remove(prefix);
return paths;
}
}
......@@ -39,7 +39,6 @@ import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.EnableTransactionManagement;
// TODO: Auto-generated Javadoc
/**
* The Class DatabaseConfig.
*/
......
......@@ -76,6 +76,7 @@ public class FileRepositoryAddTest {
final String originalFilename = "requirements" + extension;
final RepositoryFile repoFile = fileRepoService.addFile(PATH, originalFilename, contentType, SOME_BYTES, null);
assertThat(repoFile.getSize(), is(SOME_BYTES.length));
FileRepositoryTestUtil.checkFile(repoFile, PATH, originalFilename, extension, contentType);
}
......@@ -153,9 +154,10 @@ public class FileRepositoryAddTest {
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryFileDataException.class)
public void invalidMissingContentType() throws InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
fileRepoService.addFile("/valid/path/", "orignalFilename.txt", null, FileRepositoryAddTest.EMPTY_BYTES, null);
@Test
public void defaultContentType() throws InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
RepositoryFile f = fileRepoService.addFile("/valid/path/", "orignalFilename.txt", null, FileRepositoryAddTest.EMPTY_BYTES, null);
assertThat(f.getContentType(), is("application/octet-stream"));
}
/**
......
......@@ -71,26 +71,26 @@ public class FileRepositoryDirectoryTest {
final String originalFilename = "folderfile" + extension;
final List<RepositoryFile> repoFiles = new ArrayList<RepositoryFile>(20);
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 3 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 4 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 5 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/gg/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/ee/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/ee/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", 3 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", 4 + originalFilename, contentType, SOME_BYTES, null));