Commit ca5ffb2f authored by Jürgen Hannus's avatar Jürgen Hannus
Browse files

Merge branch 'develop' into 'master'

Release: merge 'develop' into 'master' created by Jürgen Hannus

See merge request teis/files-service!56
parents 4df949ee b0cefc9d
......@@ -8,7 +8,7 @@ variables:
USE_RABBIT: "true"
USE_S3: "true"
BUILD_SERVICE_PROJECT_NAME: "files-service"
BUILD_SERVICE_LIB_PROJECT_NAME: "files-service-lib files-client-lib"
BUILD_SERVICE_LIB_PROJECT_NAME: "files-service-lib"
include:
- project: "teis/dev-ops"
......
# Changelog
## [1.17.0] - 2021-03-08
* TEIS-2018: added functionality to fetch all CURRENT file references via file ID
* Updated Minio client dependency to latest version; Code refactor due to MinioClient SDK breaking changes
* TEIS-2037: added Prometheus to enable hardware monitoring
* Moved FileMetadata from files-service-lib to domain-lib
* NB: replaced Gradle compile with implementation which doesn't expose service libraries to depending services. Services
which depend on latest versions of files-service-lib may require to add dependencies for Gradle task
integTestImplementation in service Gradle configuration
## [1.16.0] - 2021-02-22
* functionality to map S3 Minio bucket dynamically (configuration provided by files service)
......
functionalities{
library
file
}
description = """files-client-lib"""
dependencies {
implementation "ee.sm.ti.teis:service-request-lib:${commonsVersion}"
implementation 'org.hibernate:hibernate-core:5.4.22.Final'
}
apply from: this.getClass().getClassLoader().getResource('teis.publishLib.gradle')
gradle.ext.artifactoryUrl = hasProperty('ARTIFACTORY_URL') ? ARTIFACTORY_URL : System.getenv('ARTIFACTORY_URL')
gradle.ext.artifactoryUser = hasProperty('ARTIFACTORY_USER') ? ARTIFACTORY_USER : System.getenv('ARTIFACTORY_USER')
gradle.ext.artifactoryPass = hasProperty('ARTIFACTORY_PASS') ? ARTIFACTORY_PASS : System.getenv('ARTIFACTORY_PASS')
gradle.ext.artifactoryRepoKey = hasProperty('ARTIFACTORY_REPO_KEY') ? ARTIFACTORY_REPO_KEY : System.getenv('ARTIFACTORY_REPO_KEY')
rootProject.name='files-client-lib'
package ee.sm.ti.teis.fileclient;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileStorageBucketsConfigurationDto {
String bucket;
String thumbnailsBucket;
List<String> objectTypes;
}
package ee.sm.ti.teis.fileclient;
import ee.sm.ti.teis.AbstractDTO;
import ee.sm.ti.teis.ErrorDTO;
public class FileStorageBucketsConfigurationRequest extends AbstractDTO<Void, ErrorDTO> {
public static final String ROUTING_KEY = "api.FileStorageBucketsConfigurationRequest";
@Override
public String routingKey() {
return ROUTING_KEY;
}
}
package ee.sm.ti.teis.fileclient;
import ee.sm.ti.teis.AbstractDTO;
import ee.sm.ti.teis.ErrorDTO;
import java.util.List;
public class FileStorageBucketsConfigurationResponse extends AbstractDTO<List<FileStorageBucketsConfigurationDto>, ErrorDTO> {
public static final String ROUTING_KEY = "api.FileStorageBucketsConfigurationResponse";
@Override
public String routingKey() {
return ROUTING_KEY;
}
}
package ee.sm.ti.teis.fileclient;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import static ee.sm.ti.teis.logging.LoggingHelper.logAdminAlertError;
import static ee.sm.ti.teis.servicerequest.UserType.SYSTEM;
import static java.util.UUID.randomUUID;
@ConditionalOnProperty(
value = {"teis.file.init-bucket-configuration"},
havingValue = "true"
)
@Component
@Slf4j
@RequiredArgsConstructor
public class FileStorageConfigService {
private static final RequestMetaDTO SYSTEM_REQUEST_META_DTO = RequestMetaDTO.builder()
.requestId(randomUUID().toString())
.userType(SYSTEM)
.build();
private final FileStorageServiceClient fileStorageServiceClient;
@EventListener(ApplicationReadyEvent.class)
public void getFileStorageBucketConfigurations() {
try {
log.info("Trying to get file storage configuration from files service");
fileStorageServiceClient.getBucketConfigurations(SYSTEM_REQUEST_META_DTO);
log.info("File storage buckets configuration received from files service successfully");
} catch (Exception e) {
log.error("Exception occurred fetching file storage configuration from files service, proceeding with application launch..");
logAdminAlertError("Retrieval of file storage configuration from files service failed with exception: ", e.getMessage());
}
}
}
package ee.sm.ti.teis.fileclient;
import ee.sm.ti.teis.ErrorDTO;
import ee.sm.ti.teis.exceptions.TeisBusinessException;
import ee.sm.ti.teis.exceptions.TeisRestException;
import ee.sm.ti.teis.exceptions.TeisStorageException;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import io.minio.MinioClient;
import io.minio.ObjectStat;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Service;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import java.io.InputStream;
import java.util.List;
import java.util.UUID;
import static ee.sm.ti.teis.errors.CommonErrorCode.SYSTEM_ERROR;
import static ee.sm.ti.teis.logging.LoggingHelper.logAdminAlertError;
import static java.util.Collections.emptyList;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import static org.springframework.util.CollectionUtils.isEmpty;
@Service
@Slf4j
@RequiredArgsConstructor
public class FileStorageServiceClient {
private final RabbitTemplate gwRabbitTemplate;
private final MinioClient minioClient;
private List<FileStorageBucketsConfigurationDto> fileStorageBucketConfigurations = emptyList();
public List<FileStorageBucketsConfigurationDto> getBucketConfigurations(RequestMetaDTO requestMetaDTO) {
FileStorageBucketsConfigurationRequest request = new FileStorageBucketsConfigurationRequest();
request.setPayload(null, requestMetaDTO);
ParameterizedTypeReference<FileStorageBucketsConfigurationResponse> responseType =
ParameterizedTypeReference.forType(FileStorageBucketsConfigurationResponse.class);
FileStorageBucketsConfigurationResponse response =
gwRabbitTemplate.convertSendAndReceiveAsType(request.routingKey(), request, responseType);
if (response != null) {
fileStorageBucketConfigurations = response.processResponse();
return fileStorageBucketConfigurations;
}
throw new TeisBusinessException(SYSTEM_ERROR, "No response from files service");
}
public String getBucket(@NotBlank String objectType, RequestMetaDTO requestMetaDTO) {
if (isEmpty(fileStorageBucketConfigurations)) {
fileStorageBucketConfigurations = getBucketConfigurations(requestMetaDTO);
}
FileStorageBucketsConfigurationDto bucketConfiguration = fileStorageBucketConfigurations.stream()
.filter(c -> c.getObjectTypes().contains(parseObjectType(objectType)))
.findFirst()
.orElseThrow(() -> new TeisRestException(ErrorDTO.builder()
.code(SYSTEM_ERROR.getCode())
.requestId(requestMetaDTO.getRequestId())
.httpResponse(INTERNAL_SERVER_ERROR)
.message("Bucket configuration not found for object type: " + objectType)
.build()));
String bucket = bucketConfiguration.getBucket();
validateCreateBucket(bucket);
return bucket;
}
private String parseObjectType(String objectType) {
return objectType.contains("__") ? objectType.substring(objectType.indexOf("__") + 2) : objectType;
}
public void validateCreateBucket(@NotBlank String bucket) {
try {
boolean bucketExists = minioClient.bucketExists(bucket);
if (!bucketExists) {
log.info("Bucket [{}] does not exist, creating provided bucket", bucket);
minioClient.makeBucket(bucket);
}
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to create new file storage bucket [{}]", e, bucket);
throw new TeisStorageException("Error occurred when creating new file storage bucket: " + e.getMessage());
}
}
public InputStream getFileStream(@NotBlank String bucket, @NotNull UUID fileName) {
return getFileStream(bucket, fileName.toString());
}
public InputStream getFileStream(@NotBlank String bucket, @NotBlank String fileName) {
InputStream fileStream;
try {
fileStream = minioClient.getObject(bucket, fileName);
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to get file {} from storage bucket [{}]", fileName, e, bucket);
throw new TeisStorageException("Error occurred when retrieving file from file storage bucket: " + e.getMessage());
}
return fileStream;
}
public void saveFile(@NotBlank String bucket, @NotNull UUID fileName, InputStream fileStream, String contentType) {
saveFile(bucket, fileName.toString(), fileStream, contentType);
}
public void saveFile(@NotBlank String bucket, @NotBlank String fileName, InputStream fileStream, String contentType) {
validateCreateBucket(bucket);
try {
minioClient.putObject(bucket, fileName,
fileStream, null, null, null, contentType);
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to save file {} to storage bucket [{}]", fileName, e, bucket);
throw new TeisStorageException("Error occurred when saving file to file storage bucket: " + e.getMessage());
}
}
public void deleteFile(@NotBlank String bucket, @NotNull UUID fileName) {
deleteFile(bucket, fileName.toString());
}
public void deleteFile(@NotBlank String bucket, @NotBlank String fileName) {
try {
minioClient.removeObject(bucket, fileName);
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to delete file {} from storage bucket [{}]", fileName, e, bucket);
throw new TeisStorageException("Error occurred when deleting file from file storage bucket: " + e.getMessage());
}
}
public ObjectStat getFileMetadata(@NotBlank String bucket, @NotNull UUID fileName) {
return getFileMetadata(bucket, fileName.toString());
}
public ObjectStat getFileMetadata(@NotBlank String bucket, @NotBlank String fileName) {
try {
return minioClient.statObject(bucket, fileName);
} catch (Exception e) {
logAdminAlertError("Exception occurred when trying to get file {} metadata from storage bucket [{}]", fileName, e, bucket);
throw new TeisStorageException("Error occurred when fetching file metadata from file storage bucket: " + e.getMessage());
}
}
}
theGroup=ee.sm.ti.teis
theVersion=1.16.2
commonsVersion=1.20.0
commonApiGatewayVersion=1.20.0
pluginVersion=1.2.0
schedulerVersion=1.13.0
theVersion=1.17.0
commonsVersion=1.21.0
commonApiGatewayVersion=1.21.0
pluginVersion=1.3.0
schedulerVersion=1.14.0
......@@ -7,11 +7,11 @@ dependencies {
implementation 'org.hibernate:hibernate-core:5.4.22.Final'
implementation 'org.springframework.data:spring-data-jpa:2.3.4.RELEASE'
// todo: refactor ClassifierService and remove
implementation "ee.sm.ti.teis:domain-cache-lib:${commonsVersion}"
// todo: refactor ClassifierService and remove
implementation "ee.sm.ti.teis:common-api-gateway-lib:${commonApiGatewayVersion}"
implementation "ee.sm.ti.teis:service-common-lib:${commonsVersion}"
testImplementation "ee.sm.ti.teis:domain-lib:${commonsVersion}"
}
apply from: this.getClass().getClassLoader().getResource('teis.publishLib.gradle')
package ee.sm.ti.teis.file;
import ee.sm.ti.teis.ErrorDTO;
import ee.sm.ti.teis.domain.file.FileMetadata;
import ee.sm.ti.teis.domainrequest.DomainResponseDTO;
import ee.sm.ti.teis.domainrequest.rabbit.TeisRabbitClient;
import ee.sm.ti.teis.file.domain.FileMetadata;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
......
package ee.sm.ti.teis.file.domain;
import ee.sm.ti.teis.domain.AbstractDomain;
import ee.sm.ti.teis.types.enums.FileScanStatusType;
import ee.sm.ti.teis.types.enums.FileStatusType;
import ee.sm.ti.teis.types.enums.ObjectStatus;
import lombok.*;
import java.time.LocalDateTime;
import java.util.Map;
import java.util.UUID;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@EqualsAndHashCode(callSuper = false)
public class FileMetadata extends AbstractDomain {
private static final long serialVersionUID = -6562235756310268014L;
private UUID fileId;
private UUID objectId;
private String fileName;
private String contentType;
private Long size;
private FileStatusType fileStatusType;
private Boolean signed;
private LocalDateTime signedAt;
private FileScanStatusType scanStatus;
private String checksum;
private String bucket;
private String thumbnailsBucket;
private Map<String, String> thumbnails;
private ObjectStatus objectStatus;
}
......@@ -3,10 +3,10 @@ package ee.sm.ti.teis.file.filereference;
import ee.sm.ti.teis.commongateway.file.FileMetadataDto;
import ee.sm.ti.teis.commongateway.file.FileReferenceDto;
import ee.sm.ti.teis.commongateway.file.FileReferenceFullDto;
import ee.sm.ti.teis.domain.file.FileMetadata;
import ee.sm.ti.teis.exceptions.TeisResourceNotFoundException;
import ee.sm.ti.teis.file.FileNotifyService;
import ee.sm.ti.teis.file.FileService;
import ee.sm.ti.teis.file.domain.FileMetadata;
import ee.sm.ti.teis.file.domain.FilePointer;
import ee.sm.ti.teis.file.domain.FilePointerUpdate;
import ee.sm.ti.teis.file.pointer.FilePointerNotifyService;
......@@ -187,4 +187,8 @@ public class FileReferenceBusinessService {
dataService.saveAll(entities);
}
public List<FileReferenceEntity> getCurrentByFileId(UUID fileId) {
return dataService.getCurrentByFileId(fileId);
}
}
......@@ -5,9 +5,9 @@ import ee.sm.ti.teis.commongateway.file.FileMetadataDto;
import ee.sm.ti.teis.commongateway.file.FileReferenceDto;
import ee.sm.ti.teis.commongateway.file.FileReferenceFullDto;
import ee.sm.ti.teis.domain.ClassifierItem;
import ee.sm.ti.teis.domain.file.FileMetadata;
import ee.sm.ti.teis.file.classifier.FileReferenceClassifierMapper;
import ee.sm.ti.teis.file.domain.File;
import ee.sm.ti.teis.file.domain.FileMetadata;
import ee.sm.ti.teis.file.domain.FilePointer;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import ee.sm.ti.teis.types.enums.classifier.CreatorType;
......
......@@ -3,8 +3,8 @@ package ee.sm.ti.teis.file.filereference;
import ee.sm.ti.teis.commongateway.file.FileMetadataDto;
import ee.sm.ti.teis.commongateway.file.FileReferenceDto;
import ee.sm.ti.teis.commongateway.file.FileReferenceFullDto;
import ee.sm.ti.teis.domain.file.FileMetadata;
import ee.sm.ti.teis.file.domain.File;
import ee.sm.ti.teis.file.domain.FileMetadata;
import ee.sm.ti.teis.file.domain.FilePointer;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
......
......@@ -5,11 +5,11 @@ import ee.sm.ti.teis.commongateway.classifier.ClassifierItemLightDto;
import ee.sm.ti.teis.commongateway.file.FileMetadataDto;
import ee.sm.ti.teis.commongateway.file.FileReferenceDto;
import ee.sm.ti.teis.domain.ClassifierItem;
import ee.sm.ti.teis.domain.file.FileMetadata;
import ee.sm.ti.teis.exceptions.TeisResourceNotFoundException;
import ee.sm.ti.teis.file.FileNotifyService;
import ee.sm.ti.teis.file.FileService;
import ee.sm.ti.teis.file.classifier.FileReferenceClassifierMapperImpl;
import ee.sm.ti.teis.file.domain.FileMetadata;
import ee.sm.ti.teis.file.domain.FilePointer;
import ee.sm.ti.teis.file.pointer.FilePointerNotifyService;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
......
......@@ -12,8 +12,8 @@ apply from: this.getClass().getClassLoader().getResource('teis.microservice.grad
dependencies {
compile project(':files-service-lib')
compile project(':files-client-lib')
compile "ee.sm.ti.teis:files-client-lib:${commonsVersion}"
compile "ee.sm.ti.teis:scheduler-service-lib:${schedulerVersion}"
compile "ee.sm.ti.teis:common-api-gateway-lib:${commonApiGatewayVersion}"
compile "ee.sm.ti.teis:service-common-lib:${commonsVersion}"
......@@ -21,6 +21,7 @@ dependencies {
compile "ee.sm.ti.teis:service-request-lib:${commonsVersion}"
implementation 'net.coobird:thumbnailator:0.4.12'
implementation "io.micrometer:micrometer-registry-prometheus"
integTestImplementation 'net.coobird:thumbnailator:0.4.12'
integTestImplementation 'org.camunda.bpm.springboot:camunda-bpm-spring-boot-starter-webapp:3.2.1'
......
......@@ -10,16 +10,18 @@ import ee.sm.ti.teis.file.FileRepository;
import ee.sm.ti.teis.file.FilesServiceAppTestBase;
import ee.sm.ti.teis.file.pointer.FilePointerEntity;
import ee.sm.ti.teis.file.pointer.FilePointerRepository;
import ee.sm.ti.teis.fileclient.FileStorageServiceClient;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import ee.sm.ti.teis.servicerequest.UserType;
import ee.sm.ti.teis.types.enums.FileStatusType;
import io.minio.MinioClient;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import javax.transaction.Transactional;
import java.io.IOException;
import java.util.UUID;
import static ee.sm.ti.teis.types.enums.FileStatusType.FAILED;
......@@ -44,8 +46,8 @@ class FileGwListenerTest extends FilesServiceAppTestBase {
@Autowired
RabbitTemplate msRabbitTemplate;
@Autowired
MinioClient minioClient;
@MockBean
FileStorageServiceClient fileStorageServiceClient;
@Autowired
FileRepository fileRepository;
......@@ -57,7 +59,7 @@ class FileGwListenerTest extends FilesServiceAppTestBase {
FileStorageUploadFailedRequest failedRequest;
@BeforeEach
void setUp() throws Exception {
void setUp() throws IOException {
requestMetaDTO = createRequestMeta(UUID.randomUUID().toString(), UserType.OFFICIAL_USER,
UUID.randomUUID().toString(), emptyList());
......@@ -68,7 +70,7 @@ class FileGwListenerTest extends FilesServiceAppTestBase {
failedRequest.setRequestMetaDTO(requestMetaDTO);
doReturn(toInputStream("magnificent test file input", UTF_8.displayName()))
.when(minioClient).getObject(anyString(), anyString());
.when(fileStorageServiceClient).getFileStream(anyString(), anyString());
}
@Test
......
package ee.sm.ti.teis.file.listener;
import ee.sm.ti.teis.domain.admin.FeatureFlag;
import ee.sm.ti.teis.domain.file.FileMetadata;
import ee.sm.ti.teis.featureflag.FeatureFlagServiceClient.FeatureFlagResponseMessage;
import ee.sm.ti.teis.file.FileEntity;
import ee.sm.ti.teis.file.FileRepository;
......@@ -10,15 +11,15 @@ import ee.sm.ti.teis.file.config.FilesQueueConfig.DeleteFileResponse;
import ee.sm.ti.teis.file.config.FilesQueueConfig.FileRequest;
import ee.sm.ti.teis.file.config.FilesQueueConfig.FileResponse;
import ee.sm.ti.teis.file.domain.DeleteFile;
import ee.sm.ti.teis.file.domain.FileMetadata;
import ee.sm.ti.teis.file.pointer.FilePointerEntity;
import ee.sm.ti.teis.file.pointer.FilePointerRepository;
import ee.sm.ti.teis.fileclient.FileStorageServiceClient;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import io.minio.MinioClient;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import javax.transaction.Transactional;
import java.util.List;
......@@ -42,8 +43,8 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
@Autowired
RabbitTemplate msRabbitTemplate;
@Autowired
MinioClient minioClient;
@MockBean
FileStorageServiceClient fileStorageServiceClient;
@Autowired
FileRepository fileRepository;
......@@ -56,7 +57,7 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
private DeleteFileRequest deleteFileRequest;
@BeforeEach
void setUp() throws Exception {
void setUp() {
requestMetaDTO = createRequestMeta(UUID.randomUUID().toString(), OFFICIAL_USER, UUID.randomUUID().toString(),
emptyList());
......@@ -66,7 +67,7 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
deleteFileRequest = new DeleteFileRequest();
deleteFileRequest.setRequestMetaDTO(requestMetaDTO);
doNothing().when(minioClient).removeObject(anyString(), anyString());
doNothing().when(fileStorageServiceClient).deleteFile(anyString(), anyString());
FeatureFlagResponseMessage featureFlagResponse = new FeatureFlagResponseMessage();
FeatureFlag payload = new FeatureFlag();
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment