Commit 85e2f15c authored by Jürgen Hannus's avatar Jürgen Hannus
Browse files

Merge branch 'develop' into 'master'

Release: merge 'develop' into 'master' created by Jürgen Hannus

See merge request teis/files-service!52
parents 90196112 01693b97
......@@ -8,7 +8,7 @@ variables:
USE_RABBIT: "true"
USE_S3: "true"
BUILD_SERVICE_PROJECT_NAME: "files-service"
BUILD_SERVICE_LIB_PROJECT_NAME: "files-service-lib"
BUILD_SERVICE_LIB_PROJECT_NAME: "files-service-lib files-client-lib"
include:
- project: "teis/dev-ops"
......
# Changelog
## [1.16.0] - 2021-02-22
* functionality to map S3 Minio bucket dynamically (configuration provided by files service)
* moved Minio client specific functionality to FileStorageServiceClient
* added new library files-client-lib. Fixed files-service dependency management
* file storage buckets configurations are now exchanged via GW exchange, instead of MS
## [1.15.0] - 2021-01-22
* added logging for file checksum calculation and thumbnail generation initiation
......
......@@ -53,10 +53,4 @@ allprojects {
}
apply from: this.getClass().getClassLoader().getResource('teis.test-sonar.gradle')
dependencies {
compile "ee.sm.ti.teis:service-common-lib:${commonsVersion}"
compile "ee.sm.ti.teis:common-api-gateway-lib:${commonApiGatewayVersion}"
compile "ee.sm.ti.teis:domain-cache-lib:${commonsVersion}"
}
}
functionalities{
library
file
}
description = """files-client-lib"""
dependencies {
implementation "ee.sm.ti.teis:service-request-lib:${commonsVersion}"
implementation 'org.hibernate:hibernate-core:5.4.22.Final'
}
apply from: this.getClass().getClassLoader().getResource('teis.publishLib.gradle')
gradle.ext.artifactoryUrl = hasProperty('ARTIFACTORY_URL') ? ARTIFACTORY_URL : System.getenv('ARTIFACTORY_URL')
gradle.ext.artifactoryUser = hasProperty('ARTIFACTORY_USER') ? ARTIFACTORY_USER : System.getenv('ARTIFACTORY_USER')
gradle.ext.artifactoryPass = hasProperty('ARTIFACTORY_PASS') ? ARTIFACTORY_PASS : System.getenv('ARTIFACTORY_PASS')
gradle.ext.artifactoryRepoKey = hasProperty('ARTIFACTORY_REPO_KEY') ? ARTIFACTORY_REPO_KEY : System.getenv('ARTIFACTORY_REPO_KEY')
rootProject.name='files-client-lib'
package ee.sm.ti.teis.fileclient;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileStorageBucketsConfigurationDto {
String bucket;
String thumbnailsBucket;
List<String> objectTypes;
}
package ee.sm.ti.teis.fileclient;
import ee.sm.ti.teis.AbstractDTO;
import ee.sm.ti.teis.ErrorDTO;
public class FileStorageBucketsConfigurationRequest extends AbstractDTO<Void, ErrorDTO> {
public static final String ROUTING_KEY = "api.FileStorageBucketsConfigurationRequest";
@Override
public String routingKey() {
return ROUTING_KEY;
}
}
package ee.sm.ti.teis.fileclient;
import ee.sm.ti.teis.AbstractDTO;
import ee.sm.ti.teis.ErrorDTO;
import java.util.List;
public class FileStorageBucketsConfigurationResponse extends AbstractDTO<List<FileStorageBucketsConfigurationDto>, ErrorDTO> {
public static final String ROUTING_KEY = "api.FileStorageBucketsConfigurationResponse";
@Override
public String routingKey() {
return ROUTING_KEY;
}
}
package ee.sm.ti.teis.fileclient;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import static ee.sm.ti.teis.logging.LoggingHelper.logAdminAlertError;
import static ee.sm.ti.teis.servicerequest.UserType.SYSTEM;
import static java.util.UUID.randomUUID;
@ConditionalOnProperty(
value = {"teis.file.init-bucket-configuration"},
havingValue = "true"
)
@Component
@Slf4j
@RequiredArgsConstructor
public class FileStorageConfigService {
private static final RequestMetaDTO SYSTEM_REQUEST_META_DTO = RequestMetaDTO.builder()
.requestId(randomUUID().toString())
.userType(SYSTEM)
.build();
private final FileStorageServiceClient fileStorageServiceClient;
@EventListener(ApplicationReadyEvent.class)
public void getFileStorageBucketConfigurations() {
try {
log.info("Trying to get file storage configuration from files service");
fileStorageServiceClient.getBucketConfigurations(SYSTEM_REQUEST_META_DTO);
log.info("File storage buckets configuration received from files service successfully");
} catch (Exception e) {
log.error("Exception occurred fetching file storage configuration from files service, proceeding with application launch..");
logAdminAlertError("Retrieval of file storage configuration from files service failed with exception: ", e.getMessage());
}
}
}
package ee.sm.ti.teis.fileclient;
import ee.sm.ti.teis.ErrorDTO;
import ee.sm.ti.teis.exceptions.TeisBusinessException;
import ee.sm.ti.teis.exceptions.TeisRestException;
import ee.sm.ti.teis.exceptions.TeisStorageException;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import io.minio.MinioClient;
import io.minio.ObjectStat;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Service;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import java.io.InputStream;
import java.util.List;
import java.util.UUID;
import static ee.sm.ti.teis.errors.CommonErrorCode.SYSTEM_ERROR;
import static ee.sm.ti.teis.logging.LoggingHelper.logAdminAlertError;
import static java.util.Collections.emptyList;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import static org.springframework.util.CollectionUtils.isEmpty;
@Service
@Slf4j
@RequiredArgsConstructor
public class FileStorageServiceClient {
private final RabbitTemplate gwRabbitTemplate;
private final MinioClient minioClient;
private List<FileStorageBucketsConfigurationDto> fileStorageBucketConfigurations = emptyList();
public List<FileStorageBucketsConfigurationDto> getBucketConfigurations(RequestMetaDTO requestMetaDTO) {
FileStorageBucketsConfigurationRequest request = new FileStorageBucketsConfigurationRequest();
request.setPayload(null, requestMetaDTO);
ParameterizedTypeReference<FileStorageBucketsConfigurationResponse> responseType =
ParameterizedTypeReference.forType(FileStorageBucketsConfigurationResponse.class);
FileStorageBucketsConfigurationResponse response =
gwRabbitTemplate.convertSendAndReceiveAsType(request.routingKey(), request, responseType);
if (response != null) {
fileStorageBucketConfigurations = response.processResponse();
return fileStorageBucketConfigurations;
}
throw new TeisBusinessException(SYSTEM_ERROR, "No response from files service");
}
public String getBucket(@NotBlank String objectType, RequestMetaDTO requestMetaDTO) {
if (isEmpty(fileStorageBucketConfigurations)) {
fileStorageBucketConfigurations = getBucketConfigurations(requestMetaDTO);
}
FileStorageBucketsConfigurationDto bucketConfiguration = fileStorageBucketConfigurations.stream()
.filter(c -> c.getObjectTypes().contains(parseObjectType(objectType)))
.findFirst()
.orElseThrow(() -> new TeisRestException(ErrorDTO.builder()
.code(SYSTEM_ERROR.getCode())
.requestId(requestMetaDTO.getRequestId())
.httpResponse(INTERNAL_SERVER_ERROR)
.message("Bucket configuration not found for object type: " + objectType)
.build()));
String bucket = bucketConfiguration.getBucket();
validateCreateBucket(bucket);
return bucket;
}
private String parseObjectType(String objectType) {
return objectType.contains("__") ? objectType.substring(objectType.indexOf("__") + 2) : objectType;
}
public void validateCreateBucket(@NotBlank String bucket) {
try {
boolean bucketExists = minioClient.bucketExists(bucket);
if (!bucketExists) {
log.info("Bucket [{}] does not exist, creating provided bucket", bucket);
minioClient.makeBucket(bucket);
}
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to create new file storage bucket [{}]", e, bucket);
throw new TeisStorageException("Error occurred when creating new file storage bucket: " + e.getMessage());
}
}
public InputStream getFileStream(@NotBlank String bucket, @NotNull UUID fileName) {
return getFileStream(bucket, fileName.toString());
}
public InputStream getFileStream(@NotBlank String bucket, @NotBlank String fileName) {
InputStream fileStream;
try {
fileStream = minioClient.getObject(bucket, fileName);
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to get file {} from storage bucket [{}]", fileName, e, bucket);
throw new TeisStorageException("Error occurred when retrieving file from file storage bucket: " + e.getMessage());
}
return fileStream;
}
public void saveFile(@NotBlank String bucket, @NotNull UUID fileName, InputStream fileStream, String contentType) {
saveFile(bucket, fileName.toString(), fileStream, contentType);
}
public void saveFile(@NotBlank String bucket, @NotBlank String fileName, InputStream fileStream, String contentType) {
validateCreateBucket(bucket);
try {
minioClient.putObject(bucket, fileName,
fileStream, null, null, null, contentType);
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to save file {} to storage bucket [{}]", fileName, e, bucket);
throw new TeisStorageException("Error occurred when saving file to file storage bucket: " + e.getMessage());
}
}
public void deleteFile(@NotBlank String bucket, @NotNull UUID fileName) {
deleteFile(bucket, fileName.toString());
}
public void deleteFile(@NotBlank String bucket, @NotBlank String fileName) {
try {
minioClient.removeObject(bucket, fileName);
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to delete file {} from storage bucket [{}]", fileName, e, bucket);
throw new TeisStorageException("Error occurred when deleting file from file storage bucket: " + e.getMessage());
}
}
public ObjectStat getFileMetadata(@NotBlank String bucket, @NotNull UUID fileName) {
return getFileMetadata(bucket, fileName.toString());
}
public ObjectStat getFileMetadata(@NotBlank String bucket, @NotBlank String fileName) {
try {
return minioClient.statObject(bucket, fileName);
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to get file {} metadata from storage bucket [{}]", fileName, e, bucket);
throw new TeisStorageException("Error occurred when fetching file metadata from file storage bucket: " + e.getMessage());
}
}
}
theGroup=ee.sm.ti.teis
theVersion=1.15.0
commonsVersion=1.18.0
commonApiGatewayVersion=1.18.0
theVersion=1.16.0
commonsVersion=1.20.0
commonApiGatewayVersion=1.20.0-SNAPSHOT
pluginVersion=1.2.0
schedulerVersion=1.13.0
......@@ -6,6 +6,12 @@ description = """files-service-lib"""
dependencies {
implementation 'org.hibernate:hibernate-core:5.4.22.Final'
implementation 'org.springframework.data:spring-data-jpa:2.3.4.RELEASE'
// todo: refactor ClassifierService and remove
implementation "ee.sm.ti.teis:domain-cache-lib:${commonsVersion}"
// todo: refactor ClassifierService and remove
implementation "ee.sm.ti.teis:common-api-gateway-lib:${commonApiGatewayVersion}"
implementation "ee.sm.ti.teis:service-common-lib:${commonsVersion}"
}
apply from: this.getClass().getClassLoader().getResource('teis.publishLib.gradle')
......@@ -27,8 +27,10 @@ public class FileMetadata extends AbstractDomain {
private FileStatusType fileStatusType;
private Boolean signed;
private LocalDateTime signedAt;
private ObjectStatus objectStatus;
private Map<String, String> thumbnails;
private FileScanStatusType scanStatus;
private String checksum;
private String bucket;
private String thumbnailsBucket;
private Map<String, String> thumbnails;
private ObjectStatus objectStatus;
}
......@@ -12,7 +12,13 @@ apply from: this.getClass().getClassLoader().getResource('teis.microservice.grad
dependencies {
compile project(':files-service-lib')
compile project(':files-client-lib')
compile "ee.sm.ti.teis:scheduler-service-lib:${schedulerVersion}"
compile "ee.sm.ti.teis:common-api-gateway-lib:${commonApiGatewayVersion}"
compile "ee.sm.ti.teis:service-common-lib:${commonsVersion}"
compile "ee.sm.ti.teis:domain-request-lib:${commonsVersion}"
compile "ee.sm.ti.teis:service-request-lib:${commonsVersion}"
implementation 'net.coobird:thumbnailator:0.4.12'
......
......@@ -5,7 +5,10 @@ import ee.sm.ti.teis.featureflag.FeatureFlagServiceClient.FeatureFlagResponseMes
import ee.sm.ti.teis.file.FileEntity;
import ee.sm.ti.teis.file.FileRepository;
import ee.sm.ti.teis.file.FilesServiceAppTestBase;
import ee.sm.ti.teis.file.config.FilesQueueConfig.*;
import ee.sm.ti.teis.file.config.FilesQueueConfig.DeleteFileRequest;
import ee.sm.ti.teis.file.config.FilesQueueConfig.DeleteFileResponse;
import ee.sm.ti.teis.file.config.FilesQueueConfig.FileRequest;
import ee.sm.ti.teis.file.config.FilesQueueConfig.FileResponse;
import ee.sm.ti.teis.file.domain.DeleteFile;
import ee.sm.ti.teis.file.domain.FileMetadata;
import ee.sm.ti.teis.file.pointer.FilePointerEntity;
......@@ -18,15 +21,10 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import javax.transaction.Transactional;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.List;
import java.util.UUID;
import static ee.sm.ti.teis.servicerequest.UserType.OFFICIAL_USER;
import static ee.sm.ti.teis.types.enums.FileScanStatusType.CLEAN;
import static ee.sm.ti.teis.types.enums.FileStatusType.ACTIVE;
import static ee.sm.ti.teis.types.enums.ObjectStatus.CURRENT;
import static ee.sm.ti.teis.types.enums.ObjectStatus.DELETED;
import static ee.sm.ti.teis.utils.TestUtils.createRequestMeta;
......@@ -97,7 +95,7 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
}
@Test
void delete_soft() throws Exception {
void delete_soft() {
DeleteFile file = new DeleteFile();
file.setFileId(UUID.fromString("3dc462d8-c0e3-44b8-80b9-4c491899cbc5"));
file.setObjectId(UUID.fromString("1a11dcca-3c35-4725-8a70-b62547981f19"));
......@@ -116,7 +114,7 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
}
@Test
void delete_soft_hasReferencingPointers_fileNotDeleted() throws Exception {
void delete_soft_hasReferencingPointers_fileNotDeleted() {
DeleteFile file = new DeleteFile();
file.setFileId(UUID.fromString("56650000-e18e-4dd8-8029-0855a9a50454"));
file.setObjectId(UUID.fromString("f315d6b5-e989-4e4b-9158-eda7eb7bc211"));
......@@ -135,7 +133,7 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
}
@Test
void delete_deleteFileWithoutPointer_success() throws Exception {
void delete_deleteFileWithoutPointer_success() {
DeleteFile file = new DeleteFile();
file.setFileId(UUID.fromString("c51c7ed0-3fb4-4bf7-9b0d-21f317286b27"));
file.setHardDelete(true);
......@@ -151,32 +149,4 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
List<FilePointerEntity> filePointers = filePointerRepository.findByFileId(file.getFileId());
assertThat(filePointers).isEmpty();
}
@Test
void createFile_success() {
CreateFileMsRequest request = new CreateFileMsRequest();
FileMetadata payload = new FileMetadata();
payload.setFileId(UUID.randomUUID());
payload.setContentType("text/plain");
payload.setFileName("test-file1.txt");
payload.setSigned(true);
payload.setSignedAt(LocalDateTime.of(LocalDate.of(2010, 5, 3), LocalTime.of(12, 13)));
payload.setSize(1253L);
request.setPayload(payload, requestMetaDTO);
CreateFileMsResponse response = listener.createFile(request);
FileMetadata domain = response.getPayload();
assertThat(domain.getFileId()).isEqualTo(payload.getFileId());
assertThat(domain.getContentType()).isEqualTo(payload.getContentType());
assertThat(domain.getFileName()).isEqualTo(payload.getFileName());
assertThat(domain.getFileStatusType()).isEqualTo(ACTIVE);
assertThat(domain.getObjectId()).isNull();
assertThat(domain.getObjectStatus()).isEqualTo(CURRENT);
assertThat(domain.getScanStatus()).isEqualTo(CLEAN);
assertThat(domain.getSigned()).isEqualTo(payload.getSigned());
assertThat(domain.getSignedAt()).isEqualTo(payload.getSignedAt());
assertThat(domain.getSize()).isEqualTo(payload.getSize());
assertThat(domain.getThumbnails()).isNull();
}
}
......@@ -35,10 +35,10 @@ import static ee.sm.ti.teis.utils.TestUtils.createRequestMeta;
import static java.util.Collections.emptyList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.*;
import static org.springframework.http.MediaType.IMAGE_JPEG_VALUE;
class FilePointerAndMetadataListenerParallelTest extends FilesServiceAppTestBase {
private static final String CONTENT_TYPE = "image/jpeg";
private static final String FILE_NAME = "test-file.jpg";
@Autowired
......@@ -89,7 +89,7 @@ class FilePointerAndMetadataListenerParallelTest extends FilesServiceAppTestBase
// create file metadata
FileMetadataDto dto = FileMetadataDto.builder()
.fileId(fileId)
.contentType(CONTENT_TYPE)
.contentType(IMAGE_JPEG_VALUE)
.size(123L)
.fileName(FILE_NAME)
.objectStatus(CURRENT)
......
......@@ -9,11 +9,14 @@ spring:
password: seis
change-log: classpath:/db/changelog/db.changelog-integtest.xml
contexts: integtest
minio:
url: http://127.0.0.1:9000
access-key: miniotest
secret-key: miniosecret
bucket: teis
teis:
file:
storage:
url: http://localhost:9000
access-key: miniotest
secret-key: miniosecret
camunda:
bpm:
admin-user:
......
......@@ -47,6 +47,8 @@ public class FileEntity extends AuditedEntity {
private String checksumAlgorithm;
private LocalDateTime checksumAt;
private Long scanStartCount;
private String bucket;
private String thumbnailsBucket;
@Column(columnDefinition = "jsonb")
@Type(type = "jsonb")
private Map<String, String> thumbnails;
......
......@@ -29,7 +29,6 @@ public interface FileMapper {
@Mapping(target = "fileName", source = "name")
@Mapping(target = "fileId", source = "id")
FileMetadata toDomain(FileEntity entity);
FileMetadataDto toDto(FileMetadata domain);
FileMetadata toDomain(FileMetadataDto dto);
......
package ee.sm.ti.teis.file;
import ee.sm.ti.teis.types.enums.FileScanStatusType;
import ee.sm.ti.teis.types.enums.FileStatusType;
import ee.sm.ti.teis.types.enums.ObjectStatus;
import org.springframework.data.jpa.repository.JpaRepository;
......@@ -52,9 +51,6 @@ public interface FileRepository extends JpaRepository<FileEntity, UUID> {
return findByChecksum(null);
}
List<FileEntity> findByScanStatusAndObjectStatusAndFileStatusAndScanStartCountLessThanEqual(FileScanStatusType scanStatus, ObjectStatus objectStatus,
FileStatusType fileStatus, Long scanStartCount);
@Query("select file " +
"from FileEntity file " +
"where (file.scanStatus is null or file.scanStatus = 'UNCHECKED') and " +
......@@ -65,7 +61,7 @@ public interface FileRepository extends JpaRepository<FileEntity, UUID> {
@Modifying
@Query("update FileEntity file " +
"set file.thumbnails = :thumbnailIds " +
"set file.thumbnails = :thumbnailIds, file.thumbnailsBucket = :thumbnailsBucket " +
"where file.id = :id")
void saveThumbnailIds(UUID id, Map<String, String> thumbnailIds);
void saveThumbnailIds(UUID id, Map<String, String> thumbnailIds, String thumbnailsBucket);
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment