Commit 929344b6 authored by Matija Obreza's avatar Matija Obreza
Browse files

Appending to file

path + originalFilename must be unique
parent 551e9e55
......@@ -22,6 +22,13 @@ package org.genesys.filerepository;
*/
public class InvalidRepositoryFileDataException extends FileRepositoryException {
public InvalidRepositoryFileDataException() {
}
public InvalidRepositoryFileDataException(String message) {
super(message);
}
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
......
......@@ -16,12 +16,18 @@
package org.genesys.filerepository;
// TODO: Auto-generated Javadoc
/**
* The Class NoSuchRepositoryFileException.
*/
public class NoSuchRepositoryFileException extends FileRepositoryException {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
public NoSuchRepositoryFileException() {
}
public NoSuchRepositoryFileException(String message) {
super(message);
}
}
......@@ -29,6 +29,7 @@ import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.persistence.Transient;
import javax.persistence.UniqueConstraint;
import org.genesys.blocks.model.AuditedVersionedModel;
import org.genesys.blocks.model.Copyable;
......@@ -39,7 +40,11 @@ import org.genesys.filerepository.metadata.BaseMetadata;
* The Class RepositoryFile.
*/
@Entity
@Table(name = "repositoryfile", indexes = { @Index(unique = false, columnList = "path", name = "IX_repoFile_path") })
@Table(name = "repositoryfile",
// indexes
indexes = { @Index(unique = false, columnList = "path", name = "IX_repoFile_path") }
// unique
, uniqueConstraints = { @UniqueConstraint(columnNames = { "path", "originalFilename" }) })
@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS)
public class RepositoryFile extends AuditedVersionedModel implements BaseMetadata, Copyable<RepositoryFile> {
......@@ -128,7 +133,7 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat
/** Byte length */
@Column(nullable = false)
private long size;
private int size;
/**
* Pre persist.
......@@ -573,14 +578,14 @@ public class RepositoryFile extends AuditedVersionedModel implements BaseMetadat
/**
* @return the size
*/
public final long getSize() {
public final int getSize() {
return size;
}
/**
* @param size the size to set
*/
public final void setSize(long size) {
public final void setSize(int size) {
this.size = size;
}
......
......@@ -75,4 +75,9 @@ public interface RepositoryFilePersistence extends JpaRepository<RepositoryFile,
@Query("select rf from RepositoryFile rf where rf.sha1Sum is null or rf.md5Sum is null")
List<RepositoryFile> findByMissingHashSums();
/**
* Find repository file by path and originalFilename
*/
RepositoryFile findByPathAndOriginalFilename(String path, String originalFilename);
}
......@@ -17,10 +17,10 @@
package org.genesys.filerepository.service.impl;
import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
......@@ -106,12 +106,7 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
PathValidator.checkValidPath(repositoryPath);
if (contentType == null) {
contentType = new Tika().detect(bytes);
} else {
String detectedContentType = new Tika().detect(bytes);
LOG.info("Content-Type provided={} detected={}", contentType, detectedContentType);
}
contentType = updateContentTypeIfNecessary(contentType, bytes);
if (originalFilename == null || contentType == null || bytes == null) {
throw new InvalidRepositoryFileDataException();
......@@ -126,11 +121,16 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
// Calculate SHA-1 and MD5 sums
repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryFile.setSize(bytes.length);
repositoryFile.setPath(repositoryPath);
repositoryFile.setOriginalFilename(originalFilename);
repositoryFile.setContentType(contentType);
if (contentType == null || contentType.length() == 0) {
throw new InvalidRepositoryFileDataException("Content type not privided and could not be detected");
}
repositoryFile = repositoryFilePersistence.save(repositoryFile);
try {
......@@ -158,12 +158,7 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
PathValidator.checkValidPath(repositoryPath);
if (contentType == null) {
contentType = new Tika().detect(bytes);
} else {
String detectedContentType = new Tika().detect(bytes);
LOG.info("Content-Type provided={} detected={}", contentType, detectedContentType);
}
contentType = updateContentTypeIfNecessary(contentType, bytes);
if (originalFilename == null || contentType == null || bytes == null) {
throw new InvalidRepositoryFileDataException();
......@@ -178,31 +173,13 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
// Calculate SHA-1 and MD5 sums
repositoryImage.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryImage.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryImage.setSize(bytes.length);
repositoryImage.setPath(repositoryPath);
repositoryImage.setOriginalFilename(originalFilename);
repositoryImage.setContentType(contentType);
try (final InputStream in = new ByteArrayInputStream(bytes)) {
in.mark(0);
final String detectedContentType = URLConnection.guessContentTypeFromStream(in);
if (detectedContentType != null && !detectedContentType.equals(contentType)) {
// overwrite contentType
if (LOG.isInfoEnabled()) {
LOG.info("Replacing content type; provided={} detected={}", contentType, detectedContentType);
}
repositoryImage.setContentType(detectedContentType);
}
in.reset();
final BufferedImage imageData = ImageIO.read(in);
if (imageData != null) {
repositoryImage.setWidth(imageData.getWidth());
repositoryImage.setHeight(imageData.getHeight());
}
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
fillImageProperties(repositoryImage, bytes);
repositoryImage = repositoryImagePersistence.save(repositoryImage);
......@@ -218,6 +195,24 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
return repositoryImage;
}
/**
* Load image to get pixel width and height
*
* @param repositoryImage
* @param bytes
*/
private void fillImageProperties(RepositoryImage repositoryImage, byte[] bytes) {
try (final InputStream in = new BufferedInputStream(new ByteArrayInputStream(bytes))) {
final BufferedImage imageData = ImageIO.read(in);
if (imageData != null) {
repositoryImage.setWidth(imageData.getWidth());
repositoryImage.setHeight(imageData.getHeight());
}
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
}
/*
* (non-Javadoc)
*
......@@ -239,13 +234,12 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
}
@Override
public RepositoryFile getFile(final String path, final String filename) throws NoSuchRepositoryFileException {
try {
UUID uuid = filename.contains(".") ? UUID.fromString(filename.substring(0, filename.indexOf('.'))) : UUID.fromString(filename);
return getFile(uuid);
} catch (IllegalArgumentException e) {
throw new NoSuchRepositoryFileException();
public RepositoryFile getFile(final String path, final String originalFilename) throws NoSuchRepositoryFileException {
RepositoryFile repositoryFile = repositoryFilePersistence.findByPathAndOriginalFilename(path, originalFilename);
if (repositoryFile == null) {
throw new NoSuchRepositoryFileException("No file at path=" + path + " originalFilename=" + originalFilename);
}
return repositoryFile;
}
/*
......@@ -335,7 +329,7 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
*/
@Override
@Transactional
public RepositoryFile updateBytes(final RepositoryFile repositoryFile, final String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException {
public RepositoryFile updateBytes(final RepositoryFile repositoryFile, String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException {
if (repositoryFile == null) {
throw new NoSuchRepositoryFileException();
}
......@@ -343,10 +337,14 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
return updateImageBytes((RepositoryImage) repositoryFile, contentType, bytes);
}
contentType = updateContentTypeIfNecessary(contentType, bytes);
// Calculate SHA-1 and MD5 sums
LOG.debug("updateByes length={}", bytes.length);
repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryFile.setSize(bytes.length);
LOG.debug("updateByes length={} repoFile.size={}", bytes.length, repositoryFile.getSize());
repositoryFile.setContentType(contentType);
bytesStorageService.upsert(repositoryFile.getPath(), repositoryFile.getFilename(), bytes);
......@@ -362,30 +360,38 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
*/
@Override
@Transactional
public RepositoryImage updateImageBytes(final RepositoryImage repositoryImage, final String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException {
public RepositoryImage updateImageBytes(final RepositoryImage repositoryImage, String contentType, final byte[] bytes) throws NoSuchRepositoryFileException, IOException {
if (repositoryImage == null) {
throw new NoSuchRepositoryFileException();
}
contentType = updateContentTypeIfNecessary(contentType, bytes);
// Calculate SHA-1 and MD5 sums
repositoryImage.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryImage.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryImage.setSize(bytes.length);
repositoryImage.setContentType(contentType);
final InputStream in = new ByteArrayInputStream(bytes);
final BufferedImage image = ImageIO.read(in);
if (image != null) {
repositoryImage.setWidth(image.getWidth());
repositoryImage.setHeight(image.getHeight());
}
fillImageProperties(repositoryImage, bytes);
bytesStorageService.upsert(repositoryImage.getPath(), repositoryImage.getFilename(), bytes);
return repositoryImagePersistence.save(repositoryImage);
}
private String updateContentTypeIfNecessary(String contentType, byte[] bytes) {
if (bytes == null) {
return contentType;
}
if (contentType == null) {
return new Tika().detect(bytes);
} else {
String detectedContentType = new Tika().detect(bytes);
LOG.info("Content-Type provided={} detected={}", contentType, detectedContentType);
return contentType;
}
}
/*
* (non-Javadoc)
*
......
......@@ -76,6 +76,7 @@ public class FileRepositoryAddTest {
final String originalFilename = "requirements" + extension;
final RepositoryFile repoFile = fileRepoService.addFile(PATH, originalFilename, contentType, SOME_BYTES, null);
assertThat(repoFile.getSize(), is(SOME_BYTES.length));
FileRepositoryTestUtil.checkFile(repoFile, PATH, originalFilename, extension, contentType);
}
......@@ -153,9 +154,10 @@ public class FileRepositoryAddTest {
* @throws InvalidRepositoryPathException the invalid repository path exception
* @throws IOException
*/
@Test(expected = InvalidRepositoryFileDataException.class)
public void invalidMissingContentType() throws InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
fileRepoService.addFile("/valid/path/", "orignalFilename.txt", null, FileRepositoryAddTest.EMPTY_BYTES, null);
@Test
public void defaultContentType() throws InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
RepositoryFile f = fileRepoService.addFile("/valid/path/", "orignalFilename.txt", null, FileRepositoryAddTest.EMPTY_BYTES, null);
assertThat(f.getContentType(), is("application/octet-stream"));
}
/**
......
......@@ -71,26 +71,26 @@ public class FileRepositoryDirectoryTest {
final String originalFilename = "folderfile" + extension;
final List<RepositoryFile> repoFiles = new ArrayList<RepositoryFile>(20);
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 3 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 4 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/", 5 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "aa/dd/ee/gg/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/ee/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/ee/", originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", 3 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", 4 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/", 5 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/ee/", 1 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/ee/", 2 + originalFilename, contentType, SOME_BYTES, null));
repoFiles.add(fileRepoService.addFile(PATH + "bb/cc/ee/ff/", originalFilename, contentType, SOME_BYTES, null));
List<String> paths = fileRepoService.listPaths(PATH + "aa/", new PageRequest(0, 10));
......
......@@ -25,4 +25,4 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %t %5p %c{1}:%L - %m
### set log levels - for more verbose logging change 'info' to 'debug' ###
log4j.rootLogger=warn, stdout
#log4j.category.org.genesys2=trace
log4j.category.org.genesys.filerepository=debug
......@@ -95,14 +95,13 @@ public class RepositoryFileSystemFactory implements FileSystemFactory, Initializ
@Override
public OutputStream createOutputStream(long offset) throws IOException {
LOG.info("Creating output stream for file={} at offset={}", getAbsolutePath(), offset);
// TODO Auto-generated method stub
return null;
return bytesManager.createOutputStream(repositoryFile, offset);
}
@Override
public InputStream createInputStream(long offset) throws IOException {
LOG.info("Creating input stream for file={} at offset={}", getAbsolutePath(), offset);
return bytesManager.createInputStream(Paths.get(getAbsolutePath()), offset);
return bytesManager.createInputStream(repositoryFile, offset);
}
};
......
......@@ -33,21 +33,19 @@ public abstract class RepositoryFtpFile implements FtpFile {
private RepositoryFile repositoryFile;
public RepositoryFtpFile() {
}
public RepositoryFtpFile(RepositoryFile repositoryFile) {
assert (repositoryFile != null);
this.repositoryFile = repositoryFile;
}
@Override
public String getAbsolutePath() {
return this.repositoryFile.getPath() + this.repositoryFile.getFilename();
return this.repositoryFile.getPath() + this.repositoryFile.getOriginalFilename();
}
@Override
public String getName() {
return this.repositoryFile.getFilename();
return this.repositoryFile.getOriginalFilename();
}
@Override
......@@ -113,7 +111,7 @@ public abstract class RepositoryFtpFile implements FtpFile {
@Override
public Object getPhysicalFile() {
// TODO Auto-generated method stub
System.err.println("getPhysicalFile??");
return null;
}
......
......@@ -88,6 +88,7 @@ public class RepositoryFtpServer implements InitializingBean, DisposableBean {
public void afterPropertiesSet() throws FtpException {
FtpServerFactory serverFactory = new FtpServerFactory();
serverFactory.setUserManager(userManager);
if (messageResource != null) {
serverFactory.setMessageResource(messageResource);
......@@ -151,6 +152,7 @@ public class RepositoryFtpServer implements InitializingBean, DisposableBean {
}
};
serverFactory.setConnectionConfig(ftpConnectionConfig);
Map<String, Ftplet> ftplets = new HashMap<>();
ftplets.put("default", repositoryFtplet());
serverFactory.setFtplets(ftplets);
......
/*
* Copyright 2017 Global Crop Diversity Trust, www.croptrust.org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.filerepository.service.ftp;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.concurrent.atomic.AtomicBoolean;
public abstract class RepositorySyncOutputStream extends BufferedOutputStream {
private AtomicBoolean oneShot = new AtomicBoolean(true);
public RepositorySyncOutputStream(OutputStream out) {
super(out);
}
@Override
public synchronized void flush() throws IOException {
super.flush();
}
@Override
public void close() throws IOException {
try {
super.close();
if (oneShot.getAndSet(false)) {
synchronizeWithRepository();
}
} finally {
cleanup();
}
}
protected abstract void cleanup() throws IOException;
protected abstract void synchronizeWithRepository() throws IOException;
}
......@@ -20,13 +20,14 @@ import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.channels.Channels;
import java.nio.file.Path;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.io.IOUtils;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
......@@ -70,90 +71,106 @@ public class TemporaryBytesManager {
File tempFile = File.createTempFile("ftp-", ".data");
return new BufferedOutputStream(new FileOutputStream(tempFile)) {
return new RepositorySyncOutputStream(new FileOutputStream(tempFile)) {
@Override
protected void cleanup() throws IOException {
LOG.info("Removing temporary file={}", tempFile.getAbsolutePath());
tempFile.delete();
}
private AtomicBoolean stored=new AtomicBoolean(false);
@Override
public void close() throws IOException {
protected void synchronizeWithRepository() throws IOException {
byte[] bytes = readTempFileToBytes(tempFile);
try {
super.close();
if (stored.getAndSet(true)) {
LOG.debug("Bytes of file={} already synchronized", tempFile.getAbsolutePath());
return;
}
String repositoryPath = parent;
String originalFilename = filename;
ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
LOG.debug("Uploaded file size={}", new File(tempFile.getAbsolutePath()).length());
try (InputStream source = new FileInputStream(tempFile)) {
IOUtils.copy(source, baos);
}
RepositoryFile metaData = new RepositoryFile();
metaData.setOriginalFilename(filename);
metaData.setPath(repositoryPath);
LOG.debug("Synchronizing file={} with repository path={} originalFilename={} size={}", tempFile.getAbsolutePath(), repositoryPath, originalFilename, baos.size());
repositoryService.addFile(repositoryPath, originalFilename, null, baos.toByteArray(), metaData);
LOG.info("Synchronized file={} with repository path={} originalFilename={}", tempFile.getAbsolutePath(), repositoryPath, originalFilename);
repositoryService</