Commit 5381c257 authored by Maxym Borodenko's avatar Maxym Borodenko

Merge branch '35-always-generate-thumbnails-for-images' into 'master'

Resolve "Always generate thumbnails for images"

Closes #35

See merge request !44
parents 30ebb831 b80d9355
Pipeline #15002 passed with stage
in 2 minutes and 48 seconds
......@@ -30,14 +30,23 @@ import org.springframework.data.domain.Pageable;
*/
public interface ImageGalleryService {
/** The Constant THUMB_PATH. */
public static final String THUMB_PATH = "/_thumbs";
/**
* The Constant THUMB_PATH.
* @deprecated Use {@link RepositoryService#THUMB_PATH}
*/
public static final String THUMB_PATH = RepositoryService.THUMB_PATH;
/** The Constant THUMB_EXT. */
public static final String THUMB_EXT = ".jpg";
/**
* The Constant THUMB_EXT.
* @deprecated Use {@link RepositoryService#THUMB_EXT}
* */
public static final String THUMB_EXT = RepositoryService.THUMB_EXT;
/** The Constant THUMB_CONTENT_TYPE. */
public static final String THUMB_CONTENT_TYPE = "image/jpeg";
/**
* The Constant THUMB_CONTENT_TYPE.
* @deprecated Use {@link RepositoryService#THUMB_CONTENT_TYPE}
*/
public static final String THUMB_CONTENT_TYPE = RepositoryService.THUMB_CONTENT_TYPE;
/**
* Loads gallery with the specified path.
......@@ -83,9 +92,10 @@ public interface ImageGalleryService {
* Delete the image gallery, but don't remove the images at that path.
*
* @param imageGallery the image gallery
* @return the removed gallery
* @throws InvalidRepositoryPathException if path is not a valid repository path
*/
void removeGallery(ImageGallery imageGallery) throws InvalidRepositoryPathException;
ImageGallery removeGallery(ImageGallery imageGallery) throws InvalidRepositoryPathException;
/**
* Update image gallery blah-blahs.
......@@ -113,17 +123,6 @@ public interface ImageGalleryService {
*/
void ensureThumbnails(ImageGallery imageGallery);
/**
* Ensure that thumbnails of images in the gallery exist in the repository at
* the {@link ImageGallery#path}/_thumb/ <code>width</code>x
* <code>height</code>_<code>uuid</code>. <code>ext</code>.
*
* @param imageGallery The ImageGallery.
* @param width Maximum width of thumbnail image. Can be null.
* @param height Maximum height of thumbnail image. Can be null.
*/
void ensureThumbnails(ImageGallery imageGallery, Integer width, Integer height);
/**
* List image galleries.
*
......
......@@ -45,6 +45,15 @@ import org.springframework.data.domain.Sort;
*/
public interface RepositoryService {
/** The Constant THUMB_PATH. */
public static final String THUMB_PATH = "/_thumbs";
/** The Constant THUMB_EXT. */
public static final String THUMB_EXT = ".jpg";
/** The Constant THUMB_CONTENT_TYPE. */
public static final String THUMB_CONTENT_TYPE = "image/jpeg";
/**
* Add a new file to the file repository.
*
......@@ -79,6 +88,13 @@ public interface RepositoryService {
RepositoryImage addImage(Path repositoryPath, String originalFilename, String contentType, byte[] bytes, RepositoryImage metaData) throws InvalidRepositoryPathException,
InvalidRepositoryFileDataException, IOException;
/**
* Generate thumbnails for the repository image if necessary.
*
* @param repositoryImage the repository image
*/
void ensureThumbnails(RepositoryImage repositoryImage);
/**
* Get repository file by its UUID.
*
......
......@@ -16,15 +16,14 @@
package org.genesys.filerepository.service.impl;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.persistence.EntityNotFoundException;
import org.genesys.blocks.security.SecurityContextUtil;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.model.ImageGallery;
......@@ -34,17 +33,14 @@ import org.genesys.filerepository.model.RepositoryFile;
import org.genesys.filerepository.model.RepositoryFolder;
import org.genesys.filerepository.model.RepositoryImage;
import org.genesys.filerepository.persistence.ImageGalleryPersistence;
import org.genesys.filerepository.service.BytesStorageService;
import org.genesys.filerepository.service.ImageGalleryService;
import org.genesys.filerepository.service.RepositoryService;
import org.genesys.filerepository.service.ThumbnailGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.access.prepost.PostAuthorize;
import org.springframework.security.access.prepost.PostFilter;
......@@ -56,8 +52,6 @@ import org.springframework.util.CollectionUtils;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import javax.persistence.EntityNotFoundException;
/**
* Image Gallery service.
*/
......@@ -76,23 +70,10 @@ public class ImageGalleryServiceImpl implements ImageGalleryService {
@Autowired
private RepositoryService repositoryService;
/** The bytes storage service. */
@Autowired
private BytesStorageService bytesStorageService;
/** Thumbnail generator. */
@Autowired
private ThumbnailGenerator thumbnailGenerator;
@Autowired(required = false)
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
/** The jpa query factory. */
@Autowired
private JPAQueryFactory jpaQueryFactory;
private int[] thumbnailSizes = { 200 };
/*
* (non-Javadoc)
* @see
......@@ -197,34 +178,13 @@ public class ImageGalleryServiceImpl implements ImageGalleryService {
@Override
@Transactional
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#imageGallery, 'delete')")
public void removeGallery(final ImageGallery imageGallery) throws InvalidRepositoryPathException {
public ImageGallery removeGallery(final ImageGallery imageGallery) throws InvalidRepositoryPathException {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting ImageGallery with id=" + imageGallery.getId());
}
for (final RepositoryImage image : imageGallery.getImages()) {
final Path imageThumbPath = getFullThumbnailsPath(image);
bytesStorageService.listFiles(imageThumbPath).forEach(filename -> {
try {
LOG.debug("Removing _thumb path={} filename={}", THUMB_PATH, filename);
bytesStorageService.remove(imageThumbPath.resolve(filename));
} catch (final Exception e) {
LOG.error("Failed to remove bytes path=" + imageThumbPath + " filename=" + filename, e);
}
});
}
imageGalleryPersistence.delete(imageGallery);
}
/**
* Path to /_thumbs/{UUID#short}/{UUID}.
*
* @param image the image
* @return the full thumbnails path
*/
public static Path getFullThumbnailsPath(final RepositoryImage image) {
return Paths.get(THUMB_PATH, image.getThumbnailPath());
return imageGallery;
}
/*
......@@ -265,8 +225,6 @@ public class ImageGalleryServiceImpl implements ImageGalleryService {
imageGallery2.setImages(imageGallery.getImages());
imageGalleryPersistence.save(imageGallery2);
ensureThumbnails(imageGallery);
return deepLoad(imageGallery2);
}
......@@ -279,129 +237,7 @@ public class ImageGalleryServiceImpl implements ImageGalleryService {
return;
}
imageGallery2.getImages().forEach(repositoryImage -> makeThumbnails(repositoryImage));
}
/**
* Use {@link #threadPoolTaskExecutor} if available.
*/
private void makeThumbnails(RepositoryImage repositoryImage) {
if (threadPoolTaskExecutor != null) {
threadPoolTaskExecutor.submit(() -> generateThumbnails(repositoryImage));
} else {
generateThumbnails(repositoryImage);
}
}
private void generateThumbnails(final RepositoryImage repositoryImage) {
try {
final byte[][] cache = new byte[thumbnailSizes.length][];
for (int i = thumbnailSizes.length - 1; i >= 0; i--) {
final int cachePos = thumbnailSizes.length - i - 1;
cache[cachePos] = ensureThumbnail(thumbnailSizes[i], thumbnailSizes[i], repositoryImage, () -> {
if (cache[cachePos] != null) {
LOG.debug("Using cached image bytes for {}", repositoryImage.getStoragePath());
return cache[cachePos];
} else {
LOG.info("Must load image bytes for {}", repositoryImage.getStoragePath());
return cache[cachePos] = bytesStorageService.get(repositoryImage.storagePath());
}
});
}
} catch (final NullPointerException e) {
LOG.error("Error generating thumbnail for image={} message={}", repositoryImage, e.getMessage(), e);
} catch (final Exception e) {
LOG.error("Error generating thumbnail for " + repositoryImage, e);
}
}
/**
* For each image in the gallery, generate a PNG thumbnail at maximum width x
* maximum height as specified. Save bytes to {@link BytesStorageService}
* directly, not through {@link RepositoryService}.
*
* Throws NotAnImageException if original bytes are not of an image,
*
* @param imageGallery the image gallery
* @param width the width
* @param height the height
* @deprecated Use {@link #ensureThumbnails(ImageGallery)} instead.
*/
@Override
public void ensureThumbnails(final ImageGallery imageGallery, final Integer width, final Integer height) {
final ImageGallery imageGallery2 = imageGalleryPersistence.findById(imageGallery.getId()).orElseThrow(() -> new EntityNotFoundException("Record not found."));
if (CollectionUtils.isEmpty(imageGallery2.getImages())) {
LOG.debug("ImageGallery has no images, skipping thumbnail generation for path=" + imageGallery2.getPath());
return;
}
imageGallery2.getImages().forEach(repositoryImage -> {
try {
ensureThumbnail(width, height, repositoryImage, () -> {
return bytesStorageService.get(repositoryImage.storagePath());
});
} catch (final NullPointerException e) {
LOG.error("Error generating thumbnail for image={} message={}", repositoryImage, e.getMessage(), e);
} catch (final Exception e) {
LOG.error("Error generating thumbnail for " + repositoryImage, e);
}
});
}
/**
* Ensure thumbnail.
*
* @param width the width
* @param height the height
* @param repositoryImage the repository image
*
* @throws IOException Signals that an I/O exception has occurred.
* @throws InvalidRepositoryPathException if path is messed up
*/
private byte[] ensureThumbnail(final Integer width, final Integer height, final RepositoryImage repositoryImage, final IImageBytesProvider loader) throws IOException, InvalidRepositoryPathException {
final String filename = getThumbnailFilename(width, height, repositoryImage.getUuid());
if (!bytesStorageService.exists(getFullThumbnailsPath(repositoryImage).resolve(filename))) {
LOG.debug("Generating new thumbnail width={} height={} for image={}", width, height, repositoryImage.getUuid());
try {
final byte[] thumbnailBytes = thumbnailGenerator.createThumbnail(width, height, loader.getImageBytes());
LOG.debug("Persisting new thumbnail width={} height={} for image={}", width, height, repositoryImage.getUuid());
bytesStorageService.upsert(getFullThumbnailsPath(repositoryImage).resolve(filename), thumbnailBytes);
return thumbnailBytes;
} catch (NullPointerException e) {
LOG.warn("Error generating thumbnail: {}", e.getMessage());
}
}
return null;
}
/**
* Gets the thumbnail filename.
*
* @param width the width
* @param height the height
* @param uuid the uuid
* @return the thumbnail filename
*/
public static final String getThumbnailFilename(final Integer width, final Integer height, final UUID uuid) {
final StringBuffer sb = new StringBuffer();
if (width != null) {
sb.append(width);
}
sb.append("x");
if (height != null) {
sb.append(height);
}
sb.append(ImageGalleryService.THUMB_EXT);
return sb.toString();
imageGallery2.getImages().forEach(repositoryImage -> repositoryService.ensureThumbnails(repositoryImage));
}
/**
......@@ -443,22 +279,4 @@ public class ImageGalleryServiceImpl implements ImageGalleryService {
return new PageImpl<>(content, pageable, total);
}
/**
* Sets the thumbnail sizes.
*
* @param thumbnailSizes the new thumbnail sizes
*/
public void setThumbnailSizes(int[] thumbnailSizes) {
Arrays.sort(thumbnailSizes);
this.thumbnailSizes = thumbnailSizes;
}
/**
* Helper to retrive image bytes (and use loaded bytes when possible).
*
* @author Matija Obreza
*/
private static interface IImageBytesProvider {
byte[] getImageBytes() throws IOException;
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
* Copyright 2020 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -22,7 +22,9 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
......@@ -53,6 +55,7 @@ import org.genesys.filerepository.persistence.RepositoryFolderRepository;
import org.genesys.filerepository.persistence.RepositoryImagePersistence;
import org.genesys.filerepository.service.BytesStorageService;
import org.genesys.filerepository.service.RepositoryService;
import org.genesys.filerepository.service.ThumbnailGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
......@@ -62,6 +65,7 @@ import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.access.prepost.PostAuthorize;
import org.springframework.security.access.prepost.PreAuthorize;
......@@ -104,11 +108,18 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/** The bytes storage service. */
@Autowired
private BytesStorageService bytesStorageService;
/** Thumbnail generator. */
@Autowired
private ThumbnailGenerator thumbnailGenerator;
/** The ACL service */
@Autowired
private CustomAclService aclService;
@Autowired(required = false)
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
/** The blacklist file names regexp */
@Value("${repository.blacklist.filename}")
private String blacklistFileNameRegex;
......@@ -123,6 +134,8 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
/** The pattern of blacklist folder names */
private Pattern blacklistFolderNamePattern;
private int[] thumbnailSizes = { 200 };
/*
* (non-Javadoc)
* @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet()
......@@ -302,7 +315,117 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
throw e;
}
return repositoryImagePersistence.save(repositoryImage);
RepositoryImage savedImage = repositoryImagePersistence.save(repositoryImage);
//
generateThumbnails(savedImage);
return savedImage;
}
/**
* Ensure thumbnails exist for the {@link RepositoryImage}
*
* @param repositoryImage the repository image
*/
@Override
public void ensureThumbnails(final RepositoryImage repositoryImage) {
if (threadPoolTaskExecutor != null) {
threadPoolTaskExecutor.submit(() -> generateThumbnails(repositoryImage));
} else {
generateThumbnails(repositoryImage);
}
}
/**
* Ensure thumbnails of configured sizes for the repository image.
*
* @param repositoryImage the repository image
*/
private void generateThumbnails(final RepositoryImage repositoryImage) {
try {
final byte[][] cache = new byte[thumbnailSizes.length][];
for (int i = thumbnailSizes.length - 1; i >= 0; i--) {
final int cachePos = thumbnailSizes.length - i - 1;
cache[cachePos] = ensureThumbnail(thumbnailSizes[i], thumbnailSizes[i], repositoryImage, () -> {
if (cache[cachePos] != null) {
LOG.debug("Using cached image bytes for {}", repositoryImage.getStoragePath());
return cache[cachePos];
} else {
LOG.info("Must load image bytes for {}", repositoryImage.getStoragePath());
return cache[cachePos] = bytesStorageService.get(repositoryImage.storagePath());
}
});
}
} catch (final NullPointerException e) {
LOG.error("Error generating thumbnail for image={} message={}", repositoryImage, e.getMessage(), e);
} catch (final Exception e) {
LOG.error("Error generating thumbnail for " + repositoryImage, e);
}
}
/**
* Ensure thumbnail.
*
* @param width the width
* @param height the height
* @param repositoryImage the repository image
*
* @throws IOException Signals that an I/O exception has occurred.
* @throws InvalidRepositoryPathException if path is messed up
*/
private byte[] ensureThumbnail(final Integer width, final Integer height, final RepositoryImage repositoryImage, final IImageBytesProvider loader) throws IOException, InvalidRepositoryPathException {
final String filename = getThumbnailFilename(width, height, repositoryImage.getUuid());
if (!bytesStorageService.exists(getFullThumbnailsPath(repositoryImage).resolve(filename))) {
LOG.debug("Generating new thumbnail width={} height={} for image={}", width, height, repositoryImage.getUuid());
try {
final byte[] thumbnailBytes = thumbnailGenerator.createThumbnail(width, height, loader.getImageBytes());
LOG.debug("Persisting new thumbnail width={} height={} for image={}", width, height, repositoryImage.getUuid());
bytesStorageService.upsert(getFullThumbnailsPath(repositoryImage).resolve(filename), thumbnailBytes);
return thumbnailBytes;
} catch (Throwable e) {
LOG.warn("Error generating thumbnail: {}", e.getMessage());
}
}
return null;
}
/**
* Path to /_thumbs/{UUID#short}/{UUID}.
*
* @param image the image
* @return the full thumbnails path
*/
public static Path getFullThumbnailsPath(final RepositoryImage image) {
return Paths.get(THUMB_PATH, image.getThumbnailPath());
}
/**
* Gets the thumbnail filename.
*
* @param width the width
* @param height the height
* @param uuid the uuid
* @return the thumbnail filename
*/
public static final String getThumbnailFilename(final Integer width, final Integer height, final UUID uuid) {
final StringBuffer sb = new StringBuffer();
if (width != null) {
sb.append(width);
}
sb.append("x");
if (height != null) {
sb.append(height);
}
sb.append(THUMB_EXT);
return sb.toString();
}
/**
......@@ -681,7 +804,7 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
bytesStorageService.remove(repositoryImage.storagePath());
Path path = ImageGalleryServiceImpl.getFullThumbnailsPath(repositoryImage);
Path path = getFullThumbnailsPath(repositoryImage);
try {
List<String> thumbnails = bytesStorageService.listFiles(path);
for (String thumb : thumbnails) {
......@@ -1016,4 +1139,22 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
}
return folder;
}
/**
* Sets the thumbnail sizes.
*
* @param thumbnailSizes the new thumbnail sizes
*/
public void setThumbnailSizes(int[] thumbnailSizes) {
Arrays.sort(thumbnailSizes);
this.thumbnailSizes = thumbnailSizes;
}
/**
* Helper to retrieve image bytes (and use loaded bytes when possible).
*/
public static interface IImageBytesProvider {
byte[] getImageBytes() throws IOException;
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
* Copyright 2020 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -32,27 +32,18 @@ import javax.imageio.ImageIO;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
import org.genesys.filerepository.model.ImageGallery;
import org.genesys.filerepository.model.RepositoryImage;
import org.genesys.filerepository.service.impl.ImageGalleryServiceImpl;
import org.junit.After;
import org.genesys.filerepository.service.impl.RepositoryServiceImpl;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.test.context.support.WithMockUser;
import org.springframework.transaction.annotation.Transactional;
// TODO: Auto-generated Javadoc
/**
* The Class ImageGalleryTest.
*/
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public class ImageGalleryThumbnailsTest extends RepositoryServiceTest {
/** The Constant DEFAULT_GALLERY_TITLE. */
private static final String DEFAULT_GALLERY_TITLE = "Test Gallery";