Commit d9623dd5 authored by Siiri Tann's avatar Siiri Tann
Browse files

TEIS-3261: updated files.calculateChecksums job logic.

parent 4ea60ff5
theGroup=ee.sm.ti.teis
theVersion=2.0.1
commonsVersion=2.2.1
commonApiGatewayVersion=2.1.0
theVersion=2.1.0-SNAPSHOT
commonsVersion=2.3.0-SNAPSHOT
commonApiGatewayVersion=2.2.0-SNAPSHOT
pluginVersion=2.1.0
schedulerVersion=1.18.0
......@@ -22,6 +22,8 @@ dependencies {
implementation "ee.sm.ti.teis:bpm-lib:${commonsVersion}"
implementation 'net.coobird:thumbnailator:0.4.12'
testImplementation('org.springframework:spring-test')
}
// Define the main class for the application
......
......@@ -6,13 +6,13 @@ import ee.sm.ti.teis.featureflag.FeatureFlagServiceClient.FeatureFlagResponseMes
import ee.sm.ti.teis.file.FileEntity;
import ee.sm.ti.teis.file.FileRepository;
import ee.sm.ti.teis.file.FilesServiceAppTestBase;
import ee.sm.ti.teis.file.config.FilesQueueConfig.DeleteFileRequest;
import ee.sm.ti.teis.file.config.FilesQueueConfig.DeleteFileResponse;
import ee.sm.ti.teis.file.config.FilesQueueConfig.FileRequest;
import ee.sm.ti.teis.file.config.FilesQueueConfig.FileResponse;
import ee.sm.ti.teis.file.config.FilesQueueConfig.*;
import ee.sm.ti.teis.file.domain.DeleteFile;
import ee.sm.ti.teis.file.hash.FileHashService;
import ee.sm.ti.teis.file.pointer.FilePointerEntity;
import ee.sm.ti.teis.file.pointer.FilePointerRepository;
import ee.sm.ti.teis.file.service.FileMetadataNotifyService.FileMetadataSavedEvent;
import ee.sm.ti.teis.file.thumbnail.FileThumbnailBusinessService;
import ee.sm.ti.teis.fileclient.FileStorageServiceClient;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import org.junit.jupiter.api.BeforeEach;
......@@ -25,11 +25,11 @@ import javax.transaction.Transactional;
import java.util.List;
import java.util.UUID;
import static ee.sm.ti.teis.servicerequest.UserType.OFFICIAL_USER;
import static ee.sm.ti.teis.types.enums.ObjectStatus.CURRENT;
import static ee.sm.ti.teis.types.enums.ObjectStatus.DELETED;
import static ee.sm.ti.teis.utils.TestUtils.createRequestMeta;
import static ee.sm.ti.teis.utils.TestUtils.createRandomOfficialUser;
import static java.util.Collections.emptyList;
import static java.util.UUID.randomUUID;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.*;
......@@ -39,27 +39,27 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
@Autowired
FileMsListener listener;
@Autowired
RabbitTemplate msRabbitTemplate;
@MockBean
FileStorageServiceClient fileStorageServiceClient;
@Autowired
FileRepository fileRepository;
@Autowired
FilePointerRepository filePointerRepository;
private RequestMetaDTO requestMetaDTO;
@MockBean
FileStorageServiceClient fileStorageServiceClient;
@MockBean
FileThumbnailBusinessService thumbnailService;
@MockBean
FileHashService hashService;
private RequestMetaDTO requestMetaDTO;
private FileRequest fileRequest;
private DeleteFileRequest deleteFileRequest;
@BeforeEach
void setUp() {
requestMetaDTO = createRequestMeta(UUID.randomUUID().toString(), OFFICIAL_USER, UUID.randomUUID().toString(),
emptyList());
requestMetaDTO = createRandomOfficialUser(emptyList());
fileRequest = new FileRequest();
fileRequest.setRequestMetaDTO(requestMetaDTO);
......@@ -154,7 +154,7 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
@Test
void delete_alreadyDeletedFileWithoutPointer_success() {
DeleteFile file = new DeleteFile();
file.setFileId(UUID.randomUUID());
file.setFileId(randomUUID());
file.setHardDelete(true);
deleteFileRequest.setPayload(file, requestMetaDTO);
......@@ -162,4 +162,35 @@ class FileMsListenerTest extends FilesServiceAppTestBase {
assertThat(response.getError()).isNull();
}
@Test
void createFile_success() {
UUID fileId = randomUUID();
CreateFileMsRequest request = new CreateFileMsRequest();
FileMetadata file = new FileMetadata();
file.setFileId(fileId);
request.setPayload(file, requestMetaDTO);
CreateFileMsResponse response = listener.createFile(request);
assertThat(response.getError()).isNull();
assertThat(response.getPayload().getFileId()).isEqualTo(fileId);
}
@Test
void updateFileMetadata_success() {
FileMetadataSavedEvent event = new FileMetadataSavedEvent();
UUID fileId = randomUUID();
FileMetadata file = new FileMetadata();
file.setFileId(fileId);
event.setPayload(file, new RequestMetaDTO());
listener.updateFileMetadata(event);
verify(hashService)
.calculateFileHash(eq(fileId), eq(null), eq(null), any(RequestMetaDTO.class));
verify(thumbnailService)
.createThumbnail(eq(file), any(RequestMetaDTO.class));
}
}
......@@ -42,13 +42,13 @@ public interface FileRepository extends JpaRepository<FileEntity, UUID> {
"(:id, :creatorType, :fileStatus, :objectStatus, NOW(), :createdBy) ON CONFLICT DO NOTHING ",
nativeQuery = true)
void createFileIfNotExists(@Param("id") UUID id, @Param("creatorType") String creatorType,
@Param("fileStatus") String fileStatus, @Param("objectStatus") String objectStatus,
@Param("createdBy") String createdBy);
@Param("fileStatus") String fileStatus, @Param("objectStatus") String objectStatus,
@Param("createdBy") String createdBy);
List<FileEntity> findByChecksum(String checksum);
List<FileEntity> findByChecksumAndObjectStatus(String checksum, ObjectStatus objectStatus);
default List<FileEntity> findByMissingChecksum() {
return findByChecksum(null);
default List<FileEntity> findCurrentByMissingChecksum() {
return findByChecksumAndObjectStatus(null, CURRENT);
}
@Query("select file " +
......
......@@ -8,6 +8,7 @@ import ee.sm.ti.teis.scheduler.job.executor.SystemJobExecutor;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
......@@ -23,9 +24,11 @@ public class UpdateFilesHashJob extends SystemJobExecutor {
private final FileDataService dataService;
private final FilesHashServiceClient filesHashServiceClient;
public UpdateFilesHashJob(RabbitTemplate msRabbitTemplate,
FileDataService dataService,
FilesHashServiceClient filesHashServiceClient) {
@Value("${teis.file.hash.algorithm}")
private String algorithm;
public UpdateFilesHashJob(RabbitTemplate msRabbitTemplate, FileDataService dataService,
FilesHashServiceClient filesHashServiceClient) {
super(msRabbitTemplate);
this.dataService = dataService;
this.filesHashServiceClient = filesHashServiceClient;
......@@ -38,7 +41,7 @@ public class UpdateFilesHashJob extends SystemJobExecutor {
@Override
public String executeInternal(SystemJob payload, RequestMetaDTO requestMetaDto) {
List<FileEntity> files = dataService.getByMissingChecksums();
List<FileEntity> files = dataService.getCurrentByMissingChecksums();
List<String> failedFileIds = new ArrayList<>();
int counter = 0;
......@@ -46,6 +49,7 @@ public class UpdateFilesHashJob extends SystemJobExecutor {
try {
CalculateFileHash domain = new CalculateFileHash();
domain.setFileId(file.getId());
domain.setAlgorithm(algorithm);
domain.setBucket(file.getBucket());
filesHashServiceClient.calculateFileHash(domain, requestMetaDto);
counter++;
......
......@@ -16,7 +16,7 @@ import javax.validation.constraints.NotNull;
import java.time.LocalDateTime;
import java.util.UUID;
import static org.springframework.util.StringUtils.isEmpty;
import static org.springframework.util.StringUtils.hasText;
@Service
@Validated
......@@ -31,7 +31,7 @@ public class FileHashService {
private String algorithm;
public void calculateFileHash(@NotNull UUID fileId, String checksum, String bucket, RequestMetaDTO requestMetaDto) {
if (isEmpty(checksum)) {
if (!hasText(checksum)) {
CalculateFileHash domain = CalculateFileHash.builder()
.fileId(fileId)
.algorithm(algorithm)
......
......@@ -59,8 +59,8 @@ public class FileDataService {
}
}
public List<FileEntity> getByMissingChecksums() {
return repository.findByMissingChecksum();
public List<FileEntity> getCurrentByMissingChecksums() {
return repository.findCurrentByMissingChecksum();
}
public List<FileEntity> getByMissingVirusScan() {
......@@ -70,4 +70,5 @@ public class FileDataService {
public void saveThumbnailIds(UUID id, Map<String, String> thumbnailIds, String thumbnailsBucket) {
repository.saveThumbnailIds(id, thumbnailIds, thumbnailsBucket);
}
}
package ee.sm.ti.teis.file;
import ee.sm.ti.teis.domain.hash.CalculateFileHash;
import ee.sm.ti.teis.exceptions.TeisBusinessException;
import ee.sm.ti.teis.file.hash.FilesHashServiceClient;
import ee.sm.ti.teis.file.service.FileDataService;
import ee.sm.ti.teis.scheduler.domain.SystemJob;
import ee.sm.ti.teis.servicerequest.RequestMetaDTO;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import java.util.List;
import static ee.sm.ti.teis.errors.CommonErrorCode.SYSTEM_ERROR;
import static java.util.Collections.emptyList;
import static java.util.UUID.randomUUID;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.*;
@ExtendWith(MockitoExtension.class)
class UpdateFilesHashJobTest {
@Mock
FileDataService dataService;
@Mock
FilesHashServiceClient filesHashServiceClient;
@InjectMocks
UpdateFilesHashJob updateFilesHashJob;
@BeforeEach
void setUp() {
ReflectionTestUtils.setField(updateFilesHashJob, "algorithm", "SHA-256");
}
@Test
void updateFilesHashJob_noFiles_success() {
doReturn(emptyList())
.when(dataService).getCurrentByMissingChecksums();
String result = updateFilesHashJob.executeInternal(new SystemJob(), new RequestMetaDTO());
assertThat(result)
.contains("File checksums updated: 0")
.doesNotContain("Failed to calculate");
verify(filesHashServiceClient, never())
.calculateFileHash(any(), any());
}
@Test
void updateFilesHashJob_hasFiles_success() {
FileEntity file = new FileEntity();
file.setId(randomUUID());
file.setBucket("bucket");
List<FileEntity> files = List.of(file);
doReturn(files)
.when(dataService).getCurrentByMissingChecksums();
String result = updateFilesHashJob.executeInternal(new SystemJob(), new RequestMetaDTO());
assertThat(result)
.contains("File checksums updated: 1")
.doesNotContain("Failed to calculate");
CalculateFileHash domain = new CalculateFileHash();
domain.setFileId(file.getId());
domain.setAlgorithm("SHA-256");
domain.setBucket(file.getBucket());
verify(filesHashServiceClient)
.calculateFileHash(eq(domain), any(RequestMetaDTO.class));
}
@Test
void updateFilesHashJob_hashServiceThrowsException_success() {
FileEntity file = new FileEntity();
file.setId(randomUUID());
file.setBucket("bucket");
List<FileEntity> files = List.of(file);
doReturn(files)
.when(dataService).getCurrentByMissingChecksums();
CalculateFileHash domain = new CalculateFileHash();
domain.setFileId(file.getId());
domain.setAlgorithm("SHA-256");
domain.setBucket(file.getBucket());
doThrow(new TeisBusinessException(SYSTEM_ERROR, "Something went wrong"))
.when(filesHashServiceClient).calculateFileHash(eq(domain), any(RequestMetaDTO.class));
String result = updateFilesHashJob.executeInternal(new SystemJob(), new RequestMetaDTO());
assertThat(result)
.contains("File checksums updated: 0")
.contains("Failed to calculate " + 1 + " file checksums for the following file IDs: " + file.getId());
}
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment