Commit 4b689a4b authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch '8-ftp-repositoryfilesystemview' into 'master'

Resolve "(ftp) RepositoryFilesystemView"

Closes #8

See merge request !6
parents d24b7510 929344b6
...@@ -6,3 +6,4 @@ ...@@ -6,3 +6,4 @@
target target
doc doc
data data
ftpserver.jks
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
<properties> <properties>
<commons.io.version>2.4</commons.io.version> <commons.io.version>2.4</commons.io.version>
<tika.version>1.14</tika.version>
</properties> </properties>
<build> <build>
...@@ -59,6 +60,18 @@ ...@@ -59,6 +60,18 @@
</dependency> </dependency>
</dependencies> </dependencies>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins> </plugins>
</build> </build>
...@@ -159,5 +172,10 @@ ...@@ -159,5 +172,10 @@
<version>${querydsl.version}</version> <version>${querydsl.version}</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>${tika.version}</version>
</dependency>
</dependencies> </dependencies>
</project> </project>
...@@ -22,6 +22,13 @@ package org.genesys.filerepository; ...@@ -22,6 +22,13 @@ package org.genesys.filerepository;
*/ */
public class InvalidRepositoryFileDataException extends FileRepositoryException { public class InvalidRepositoryFileDataException extends FileRepositoryException {
public InvalidRepositoryFileDataException() {
}
public InvalidRepositoryFileDataException(String message) {
super(message);
}
/** The Constant serialVersionUID. */ /** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
......
...@@ -16,12 +16,18 @@ ...@@ -16,12 +16,18 @@
package org.genesys.filerepository; package org.genesys.filerepository;
// TODO: Auto-generated Javadoc
/** /**
* The Class NoSuchRepositoryFileException. * The Class NoSuchRepositoryFileException.
*/ */
public class NoSuchRepositoryFileException extends FileRepositoryException { public class NoSuchRepositoryFileException extends FileRepositoryException {
/** The Constant serialVersionUID. */ /** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public NoSuchRepositoryFileException() {
}
public NoSuchRepositoryFileException(String message) {
super(message);
}
} }
...@@ -29,6 +29,7 @@ import javax.persistence.Table; ...@@ -29,6 +29,7 @@ import javax.persistence.Table;
import javax.persistence.Temporal; import javax.persistence.Temporal;
import javax.persistence.TemporalType; import javax.persistence.TemporalType;
import javax.persistence.Transient; import javax.persistence.Transient;
import javax.persistence.UniqueConstraint;
import org.genesys.blocks.model.AuditedVersionedModel; import org.genesys.blocks.model.AuditedVersionedModel;
import org.genesys.blocks.model.Copyable; import org.genesys.blocks.model.Copyable;
...@@ -39,7 +40,11 @@ import org.genesys.filerepository.metadata.BaseMetadata; ...@@ -39,7 +40,11 @@ import org.genesys.filerepository.metadata.BaseMetadata;
* The Class RepositoryFile. * The Class RepositoryFile.
*/ */
@Entity @Entity
@Table(name = "repositoryfile", indexes = { @Index(unique = false, columnList = "path", name = "IX_repoFile_path") }) @Table(name = "repositoryfile",
// indexes
indexes = { @Index(unique = false, columnList = "path", name = "IX_repoFile_path") }
// unique
, uniqueConstraints = { @UniqueConstraint(columnNames = { "path", "originalFilename" }) })
@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS) @Inheritance(strategy = InheritanceType.TABLE_PER_CLASS)
public class RepositoryFile extends AuditedVersionedModel implements BaseMetadata, Copyable<RepositoryFile> { public class RepositoryFile extends AuditedVersionedModel implements BaseMetadata, Copyable<RepositoryFile> {
...@@ -119,15 +124,17 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat ...@@ -119,15 +124,17 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat
private Date dateRetrieved; private Date dateRetrieved;
/** Sha1sum hash of the bytes. */ /** Sha1sum hash of the bytes. */
// TODO Update to nullable = false in next release @Column(length = 40, nullable = false)
@Column(length = 40, nullable = true)
private String sha1Sum; private String sha1Sum;
/** MD5 hash of the bytes. */ /** MD5 hash of the bytes. */
// TODO Update to nullable = false in next release @Column(length = 32, nullable = false)
@Column(length = 32, nullable = true)
private String md5Sum; private String md5Sum;
/** Byte length */
@Column(nullable = false)
private int size;
/** /**
* Pre persist. * Pre persist.
*/ */
...@@ -568,6 +575,20 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat ...@@ -568,6 +575,20 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat
this.md5Sum = md5Sum; this.md5Sum = md5Sum;
} }
/**
* @return the size
*/
public final int getSize() {
return size;
}
/**
* @param size the size to set
*/
public final void setSize(int size) {
this.size = size;
}
@Override @Override
public RepositoryFile apply(RepositoryFile source) { public RepositoryFile apply(RepositoryFile source) {
...@@ -590,6 +611,7 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat ...@@ -590,6 +611,7 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat
this.subject = source.subject; this.subject = source.subject;
this.title = source.title; this.title = source.title;
this.uuid = source.uuid; this.uuid = source.uuid;
this.size = source.size;
return this; return this;
} }
......
...@@ -58,6 +58,15 @@ public interface RepositoryFilePersistence extends JpaRepository<RepositoryFile, ...@@ -58,6 +58,15 @@ public interface RepositoryFilePersistence extends JpaRepository<RepositoryFile,
@Query("select distinct(rf.path) from RepositoryFile rf where rf.path like ?1%") @Query("select distinct(rf.path) from RepositoryFile rf where rf.path like ?1%")
List<String> listDistinctPaths(String prefix, Pageable pageable); List<String> listDistinctPaths(String prefix, Pageable pageable);
/**
* List all distinct paths (FTP)
*
* @param prefix the prefix
* @return the list of paths
*/
@Query("select distinct(rf.path) from RepositoryFile rf where rf.path like ?1%")
List<String> listDistinctPaths(String prefix);
/** /**
* Find files with missing hashes * Find files with missing hashes
* *
...@@ -65,4 +74,10 @@ public interface RepositoryFilePersistence extends JpaRepository<RepositoryFile, ...@@ -65,4 +74,10 @@ public interface RepositoryFilePersistence extends JpaRepository<RepositoryFile,
*/ */
@Query("select rf from RepositoryFile rf where rf.sha1Sum is null or rf.md5Sum is null") @Query("select rf from RepositoryFile rf where rf.sha1Sum is null or rf.md5Sum is null")
List<RepositoryFile> findByMissingHashSums(); List<RepositoryFile> findByMissingHashSums();
/**
* Find repository file by path and originalFilename
*/
RepositoryFile findByPathAndOriginalFilename(String path, String originalFilename);
} }
...@@ -75,6 +75,16 @@ public interface RepositoryService { ...@@ -75,6 +75,16 @@ public interface RepositoryService {
*/ */
RepositoryFile getFile(UUID fileUuid) throws NoSuchRepositoryFileException; RepositoryFile getFile(UUID fileUuid) throws NoSuchRepositoryFileException;
/**
* Get repository file by its path and filename
*
* @param path
* @param filename
* @return
* @throws NoSuchRepositoryFileException
*/
RepositoryFile getFile(String path, String filename) throws NoSuchRepositoryFileException;
/** /**
* Get repository file bytes by its path and filename. * Get repository file bytes by its path and filename.
* *
...@@ -174,6 +184,14 @@ public interface RepositoryService { ...@@ -174,6 +184,14 @@ public interface RepositoryService {
*/ */
List<String> listPaths(String prefix, Pageable pageRequest); List<String> listPaths(String prefix, Pageable pageRequest);
/**
* List all paths
*
* @param prefix
* @return
*/
List<String> listPaths(String prefix);
/** /**
* Update image metadata. The update is based on the record UUID. * Update image metadata. The update is based on the record UUID.
* *
......
...@@ -17,10 +17,10 @@ ...@@ -17,10 +17,10 @@
package org.genesys.filerepository.service.impl; package org.genesys.filerepository.service.impl;
import java.awt.image.BufferedImage; import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
...@@ -28,6 +28,7 @@ import java.util.UUID; ...@@ -28,6 +28,7 @@ import java.util.UUID;
import javax.imageio.ImageIO; import javax.imageio.ImageIO;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.tika.Tika;
import org.genesys.filerepository.InvalidRepositoryFileDataException; import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException; import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException; import org.genesys.filerepository.NoSuchRepositoryFileException;
...@@ -95,16 +96,18 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -95,16 +96,18 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
* @see org.genesys.filerepository.service.RepositoryService#addFile(java .lang.String, java.lang.String, java.lang.String, byte[], * @see org.genesys.filerepository.service.RepositoryService#addFile(java .lang.String, java.lang.String,
* org.genesys.filerepository.model.RepositoryFile) * java.lang.String, byte[], org.genesys.filerepository.model.RepositoryFile)
*/ */
@Override @Override
@Transactional @Transactional
public RepositoryFile addFile(final String repositoryPath, final String originalFilename, final String contentType, final byte[] bytes, final RepositoryFile metaData) public RepositoryFile addFile(final String repositoryPath, final String originalFilename, String contentType, final byte[] bytes, final RepositoryFile metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException { throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
PathValidator.checkValidPath(repositoryPath); PathValidator.checkValidPath(repositoryPath);
contentType = updateContentTypeIfNecessary(contentType, bytes);
if (originalFilename == null || contentType == null || bytes == null) { if (originalFilename == null || contentType == null || bytes == null) {
throw new InvalidRepositoryFileDataException(); throw new InvalidRepositoryFileDataException();
} }
...@@ -118,11 +121,16 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -118,11 +121,16 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
// Calculate SHA-1 and MD5 sums // Calculate SHA-1 and MD5 sums
repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes)); repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes)); repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryFile.setSize(bytes.length);
repositoryFile.setPath(repositoryPath); repositoryFile.setPath(repositoryPath);
repositoryFile.setOriginalFilename(originalFilename); repositoryFile.setOriginalFilename(originalFilename);
repositoryFile.setContentType(contentType); repositoryFile.setContentType(contentType);
if (contentType == null || contentType.length() == 0) {
throw new InvalidRepositoryFileDataException("Content type not privided and could not be detected");
}
repositoryFile = repositoryFilePersistence.save(repositoryFile); repositoryFile = repositoryFilePersistence.save(repositoryFile);
try { try {
...@@ -140,16 +148,18 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -140,16 +148,18 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
* @see org.genesys.filerepository.service.RepositoryService#addImage( java.lang.String, java.lang.String, java.lang.String, byte[], * @see org.genesys.filerepository.service.RepositoryService#addImage( java.lang.String, java.lang.String,
* org.genesys.filerepository.model.RepositoryImage) * java.lang.String, byte[], org.genesys.filerepository.model.RepositoryImage)
*/ */
@Override @Override
@Transactional @Transactional
public RepositoryImage addImage(final String repositoryPath, final String originalFilename, final String contentType, final byte[] bytes, final RepositoryImage metaData) public RepositoryImage addImage(final String repositoryPath, final String originalFilename, String contentType, final byte[] bytes, final RepositoryImage metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException { throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
PathValidator.checkValidPath(repositoryPath); PathValidator.checkValidPath(repositoryPath);
contentType = updateContentTypeIfNecessary(contentType, bytes);
if (originalFilename == null || contentType == null || bytes == null) { if (originalFilename == null || contentType == null || bytes == null) {
throw new InvalidRepositoryFileDataException(); throw new InvalidRepositoryFileDataException();
} }
...@@ -163,31 +173,13 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -163,31 +173,13 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
// Calculate SHA-1 and MD5 sums // Calculate SHA-1 and MD5 sums
repositoryImage.setSha1Sum(DigestUtils.sha1Hex(bytes)); repositoryImage.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryImage.setMd5Sum(DigestUtils.md5Hex(bytes)); repositoryImage.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryImage.setSize(bytes.length);
repositoryImage.setPath(repositoryPath); repositoryImage.setPath(repositoryPath);
repositoryImage.setOriginalFilename(originalFilename); repositoryImage.setOriginalFilename(originalFilename);
repositoryImage.setContentType(contentType); repositoryImage.setContentType(contentType);
try (final InputStream in = new ByteArrayInputStream(bytes)) { fillImageProperties(repositoryImage, bytes);
in.mark(0);
final String detectedContentType = URLConnection.guessContentTypeFromStream(in);
if (detectedContentType != null && !detectedContentType.equals(contentType)) {
// overwrite contentType
if (LOG.isInfoEnabled()) {
LOG.info("Replacing content type; provided={} detected={}", contentType, detectedContentType);
}
repositoryImage.setContentType(detectedContentType);
}
in.reset();
final BufferedImage imageData = ImageIO.read(in);
if (imageData != null) {
repositoryImage.setWidth(imageData.getWidth());
repositoryImage.setHeight(imageData.getHeight());
}
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
repositoryImage = repositoryImagePersistence.save(repositoryImage); repositoryImage = repositoryImagePersistence.save(repositoryImage);
...@@ -203,6 +195,24 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -203,6 +195,24 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
return repositoryImage; return repositoryImage;
} }
/**
* Load image to get pixel width and height
*
* @param repositoryImage
* @param bytes
*/
private void fillImageProperties(RepositoryImage repositoryImage, byte[] bytes) {
try (final InputStream in = new BufferedInputStream(new ByteArrayInputStream(bytes))) {
final BufferedImage imageData = ImageIO.read(in);
if (imageData != null) {
repositoryImage.setWidth(imageData.getWidth());
repositoryImage.setHeight(imageData.getHeight());
}
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
}
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
...@@ -223,6 +233,15 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -223,6 +233,15 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
return file; return file;
} }
@Override
public RepositoryFile getFile(final String path, final String originalFilename) throws NoSuchRepositoryFileException {
RepositoryFile repositoryFile = repositoryFilePersistence.findByPathAndOriginalFilename(path, originalFilename);
if (repositoryFile == null) {
throw new NoSuchRepositoryFileException("No file at path=" + path + " originalFilename=" + originalFilename);
}
return repositoryFile;
}
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
...@@ -262,7 +281,8 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -262,7 +281,8 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
* @see org.genesys.filerepository.service.RepositoryService#updateFile( org.genesys.filerepository.model .RepositoryFile) * @see org.genesys.filerepository.service.RepositoryService#updateFile( org.genesys.filerepository.model
* .RepositoryFile)
*/ */
@Override @Override
@Transactional @Transactional
...@@ -284,7 +304,8 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -284,7 +304,8 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
* @see org.genesys.filerepository.service.RepositoryService#updateMetadata(org.genesys.filerepository. model.RepositoryImage) * @see org.genesys.filerepository.service.RepositoryService#updateMetadata(org.genesys.filerepository.
* model.RepositoryImage)
*/ */
@Override @Override
@Transactional @Transactional
...@@ -303,11 +324,12 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -303,11 +324,12 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
* @see org.genesys.filerepository.service.RepositoryService#updateBytes( org.genesys.filerepository.model .RepositoryFile, java.lang.String, byte[]) * @see org.genesys.filerepository.service.RepositoryService#updateBytes( org.genesys.filerepository.model
* .RepositoryFile, java.lang.String, byte[])
*/ */
@Override @Override
@Transactional @Transactional
public RepositoryFile updateBytes(final RepositoryFile repositoryFile, final String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException { public RepositoryFile updateBytes(final RepositoryFile repositoryFile, String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException {
if (repositoryFile == null) { if (repositoryFile == null) {
throw new NoSuchRepositoryFileException(); throw new NoSuchRepositoryFileException();
} }
...@@ -315,9 +337,14 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -315,9 +337,14 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
return updateImageBytes((RepositoryImage) repositoryFile, contentType, bytes); return updateImageBytes((RepositoryImage) repositoryFile, contentType, bytes);
} }
contentType = updateContentTypeIfNecessary(contentType, bytes);
// Calculate SHA-1 and MD5 sums // Calculate SHA-1 and MD5 sums
LOG.debug("updateByes length={}", bytes.length);
repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes)); repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes)); repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryFile.setSize(bytes.length);
LOG.debug("updateByes length={} repoFile.size={}", bytes.length, repositoryFile.getSize());
repositoryFile.setContentType(contentType); repositoryFile.setContentType(contentType);
bytesStorageService.upsert(repositoryFile.getPath(), repositoryFile.getFilename(), bytes); bytesStorageService.upsert(repositoryFile.getPath(), repositoryFile.getFilename(), bytes);
...@@ -328,38 +355,48 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea ...@@ -328,38 +355,48 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
* @see org.genesys.filerepository.service.RepositoryService#updateBytes( org.genesys.filerepository.model .RepositoryImage, java.lang.String, byte[]) * @see org.genesys.filerepository.service.RepositoryService#updateBytes( org.genesys.filerepository.model