Commit 94b1ddeb authored by oleg.pahhomov's avatar oleg.pahhomov

SA0140-380, SA0140-381 fix diagram without data exceptions. Add excel exception

parent a07fe877
......@@ -9,7 +9,9 @@ import ee.stat.dashboard.repository.ExcelDataRepository;
import ee.stat.dashboard.repository.ExcelRepository;
import ee.stat.dashboard.service.admin.widget.dto.ExcelAdminDto;
import ee.stat.dashboard.service.admin.widget.dto.WidgetAdminResponse;
import ee.stat.dashboard.service.widget.coordinates.ExcelImportException;
import ee.stat.dashboard.service.widget.coordinates.ExcelToCoordinates;
import ee.stat.dashboard.util.ResponseUtil;
import lombok.AllArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
......@@ -44,7 +46,13 @@ public class ExcelAdminService {
ExcelData data = new ExcelData();
data.setExcel(excel.getId());
Arrays.stream(Language.values())
.forEach(lang -> data.setResponse(excelToCoordinates.convert(excel, lang), lang));
.forEach(lang -> {
try {
data.setResponse(excelToCoordinates.convert(excel, lang), lang);
} catch (ExcelImportException e) {
throw ResponseUtil.badRequest("excel", e.getMessage()).get();
}
});
excelDataRepository.save(data);
}
if (existingExcel != null){
......
package ee.stat.dashboard.service.widget.coordinates;
public class ExcelImportException extends Exception{
public ExcelImportException(String message) {
super(message);
}
}
......@@ -37,18 +37,19 @@ public class ExcelToCoordinates {
private CommonExcelMapper commonExcelMapper;
private ExcelSplitter excelSplitter;
public CoordinatedResponses convert(Excel excel, Language lang) {
public CoordinatedResponses convert(Excel excel, Language lang) throws ExcelImportException {
Workbook workbook = getSheets(excel, lang);
if (workbook == null) return null;
List<List<String>> rows = commonExcelMapper.excelToStrings(workbook.getSheetAt(0));
return convertExcelRows(excel, lang, rows);
}
private CoordinatedResponses convertExcelRows(Excel excel, Language lang, List<List<String>> rows) {
List<List<List<String>>> tables = partition(excel, rows);
List<CoordinatedResponse> excelResponses = tables.stream()
.map(rows1 -> buildResponse(rows1, lang))
.collect(Collectors.toList());
private CoordinatedResponses convertExcelRows(Excel excel, Language lang, List<List<String>> rows) throws ExcelImportException {
List<CoordinatedResponse> excelResponses = new ArrayList<>();
for (List<List<String>> rows1 : partition(excel, rows)) {
CoordinatedResponse coordinatedResponse = buildResponse(rows1, lang);
excelResponses.add(coordinatedResponse);
}
return new CoordinatedResponses(excelResponses);
}
......@@ -59,14 +60,14 @@ public class ExcelToCoordinates {
return excelSplitter.toWidgetExcelPartition(rows).getData();
}
private CoordinatedResponse buildResponse(List<List<String>> rows, Language lang) {
private CoordinatedResponse buildResponse(List<List<String>> rows, Language lang) throws ExcelImportException {
CoordinatedResponse response = new CoordinatedResponse();
List<DataPoint> dataPoints = buildDataPoints(rows, lang, response);
response.setDataPoints(dataPoints);
return response;
}
private List<DataPoint> buildDataPoints(List<List<String>> rows, Language lang, CoordinatedResponse response) {
private List<DataPoint> buildDataPoints(List<List<String>> rows, Language lang, CoordinatedResponse response) throws ExcelImportException {
for (List<String> row : rows) {
if (row.get(0).equals(excelConfig.getType())) {
response.setSource(Source.getFromFile(row.get(1), excelConfig.getRealLevels()));
......@@ -137,7 +138,7 @@ public class ExcelToCoordinates {
return CommonExcelMapper.isDouble(row.get(j));
}
private int dataStartColumn(List<String> dataHeader) {
private int dataStartColumn(List<String> dataHeader) throws ExcelImportException {
int dataColumnIndex = 0;
for (int i = 0; i < dataHeader.size(); i++) {
if (isNum(dataHeader, i)) {
......@@ -146,14 +147,14 @@ public class ExcelToCoordinates {
}
}
if (isEmpty(dataHeader) || dataColumnIndex == 0 && !isNum(dataHeader, 0)) {
throw new UnsupportedOperationException("data not found");
throw new ExcelImportException("data not found");
}
return dataColumnIndex;
}
private LocalDate pickDate(String dateColumn) {
private LocalDate pickDate(String dateColumn) throws ExcelImportException {
if (dateColumn.length() != 6) {
throw new UnsupportedOperationException("unknown date column length: " + dateColumn);
throw new ExcelImportException("unknown date column length: " + dateColumn);
}
return LocalDate.of(Double.valueOf(dateColumn).intValue(), 1, 1);
}
......@@ -164,9 +165,9 @@ public class ExcelToCoordinates {
return split[languagePos].trim();
}
private Workbook getSheets(Excel excel, Language lang) {
private Workbook getSheets(Excel excel, Language lang) throws ExcelImportException {
if (excel.getLocation() == null) {
throw new RuntimeException("Excel has no location, filename: " + excel.getFilename());
throw new ExcelImportException("Excel has no location, filename: " + excel.getFilename());
}
log.info("looking for excel: {} {}", lang.name(), excel.getLocation());
return commonExcelMapper.tryToCreateWorkbook(new File(excel.getLocation()));
......
......@@ -15,7 +15,7 @@ import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import static ee.stat.dashboard.util.StatDateUtil.getLatest;
import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
@Slf4j
@Service
......@@ -55,7 +55,7 @@ public class DiagramBuilder {
}
private void saveDiagram(Diagram diagram) {
if (diagram != null) {
if (diagram != null && isNotEmpty(diagram.getDiagramData())) {
Diagram savedDiagram = diagramRepository.save(diagram);
diagram.getDiagramData().forEach(e -> e.setDiagram(savedDiagram.getId()));
diagramDataRepository.saveAll(diagram.getDiagramData());
......
......@@ -8,6 +8,7 @@ import ee.stat.dashboard.model.widget.back.ExcelData;
import ee.stat.dashboard.model.widget.back.StatDb;
import ee.stat.dashboard.model.widget.back.Widget;
import ee.stat.dashboard.model.widget.back.enums.ExcelType;
import ee.stat.dashboard.service.widget.coordinates.ExcelImportException;
import ee.stat.dashboard.service.widget.coordinates.ExcelToCoordinates;
import ee.stat.dashboard.service.widget.excelsplitter.dto.WidgetExcelPartition;
import ee.stat.dashboard.service.statdata.StatDimensionsService;
......@@ -67,7 +68,13 @@ public class PartitionToEntitiesMapper {
excel.setExcelType(ExcelType.WIDGET_META_AND_DATA);
ExcelData data = new ExcelData();
Arrays.stream(Language.values())
.forEach(lang -> data.setResponse(excelToCoordinates.convert(excel, lang), lang));
.forEach(lang -> {
try {
data.setResponse(excelToCoordinates.convert(excel, lang), lang);
} catch (ExcelImportException e) {
throw new RuntimeException(e.getMessage(), e);
}
});
excel.setExcelData(Lists.newArrayList(data));
} else {
excel.setExcelType(ExcelType.WIDGET_META);
......
......@@ -25,7 +25,6 @@ public class WidgetImporter {
private PartitionToEntitiesMapper partitionToEntitiesMapper;
private WidgetSaveImporter widgetSaveImporter;
@Transactional(propagation = Propagation.REQUIRES_NEW)
public Long importWidget(File file) {
Workbook workbook = commonExcelMapper.tryToCreateWorkbook(file);
......
......@@ -25,7 +25,7 @@ public class DiagramService {
if (diagramOp.isPresent()) {
return diagramOp.get();
} else {
log.info("Diagram is missing, should not happen");
log.error("Diagram is missing, should not happen {}", widget.getId());
return null;
}
}
......@@ -34,7 +34,9 @@ public class DiagramService {
Optional<DiagramData> graphOp = diagramServiceCache.findDiagramGraph(diagramId, graphTypeEnum);
if (graphOp.isPresent()) {
return graphOp.get();
} else {
log.error("Diagram is missing, should not happen widget {} diagram {} type {}", widgetId, diagramId, graphTypeEnum);
return null;
}
throw badRequest(widgetId, "Graph missing in db").get();
}
}
......@@ -95,6 +95,10 @@ public class WidgetService {
}
DiagramData data = diagramService.findDiagramData(dto.getId(), diagram.getId(), dto.getGraphType());
if (data == null) {
dto.setDiagram(GraphDto.empty());
return dto;
}
GraphType graphType = getGraphType(graphTypeEnum, graphTypes);
......
......@@ -2,6 +2,7 @@ package ee.stat.dashboard.service.excel;
import ee.stat.dashboard.controller.publics.dto.Language;
import ee.stat.dashboard.model.widget.back.Excel;
import ee.stat.dashboard.service.widget.coordinates.ExcelImportException;
import ee.stat.dashboard.service.widget.coordinates.ExcelToCoordinates;
import org.junit.Test;
import org.junit.runner.RunWith;
......@@ -25,35 +26,35 @@ public class ExcelToCoordinatesTest {
private ExcelToCoordinates converter;
@Test
public void convert_from_data_reality_only() {
public void convert_from_data_reality_only() throws ExcelImportException {
Excel excel = new Excel("./excel/data/abs_poverty_reality.xlsx", DATA);
validateAbsPovertyReality(Language.ET, converter.convert(excel, Language.ET));
validateAbsPovertyReality(Language.EN, converter.convert(excel, Language.EN));
}
@Test
public void convert_from_data_expectations_only() {
public void convert_from_data_expectations_only() throws ExcelImportException {
Excel excel = new Excel("./excel/data/abs_poverty_expect.xlsx", DATA);
validateAbsPovertyExpectations(Language.ET, converter.convert(excel, Language.ET));
validateAbsPovertyExpectations(Language.EN, converter.convert(excel, Language.EN));
}
@Test
public void convert_from_data_expectations_and_reality() {
public void convert_from_data_expectations_and_reality() throws ExcelImportException {
Excel excel = new Excel("./excel/data/abs_poverty_expect_and_reality.xlsx", DATA);
validateAbsPovertyExpectationsAndReality(Language.ET, converter.convert(excel, Language.ET));
validateAbsPovertyExpectationsAndReality(Language.EN, converter.convert(excel, Language.EN));
}
@Test
public void convert_from_widget_expectations_excel() {
public void convert_from_widget_expectations_excel() throws ExcelImportException {
Excel excel = new Excel("./excel/widget/Noorte töötuse määr 15-24-aastaste vanusegrupis.xlsx", WIDGET_META_AND_DATA);
validateYouthUnemployment(Language.ET, converter.convert(excel, Language.ET));
validateYouthUnemployment(Language.EN, converter.convert(excel, Language.EN));
}
@Test
public void convert_from_widget_expectations_and_reality_excel() {
public void convert_from_widget_expectations_and_reality_excel() throws ExcelImportException {
Excel excel = new Excel("./excel/widget/Tööelu kestus, aastates.xlsx", WIDGET_META_AND_DATA);
validateWorkLifeExpectancy(Language.ET, converter.convert(excel, Language.ET));
validateWorkLifeExpectancy(Language.EN, converter.convert(excel, Language.EN));
......
......@@ -5,6 +5,7 @@ import ee.stat.dashboard.model.json.CoordinatedResponse;
import ee.stat.dashboard.model.json.CoordinatedResponses;
import ee.stat.dashboard.model.json.DataPoint;
import ee.stat.dashboard.model.widget.back.Excel;
import ee.stat.dashboard.service.widget.coordinates.ExcelImportException;
import ee.stat.dashboard.service.widget.coordinates.ExcelToCoordinates;
import ee.stat.dashboard.service.widget.coordinates.ResponseMerger;
import org.junit.Test;
......@@ -38,7 +39,7 @@ public class ResponseMergerTest {
private ExcelToCoordinates converter;
@Test
public void when_there_is_real_data_after_expectations_expectations_are_ignored() {
public void when_there_is_real_data_after_expectations_expectations_are_ignored() throws ExcelImportException {
Excel excel = new Excel("./excel/data/abs_poverty_expect_and_reality.xlsx", DATA);
Language lang = Language.ET;
CoordinatedResponses responses = converter.convert(excel, lang);
......@@ -50,7 +51,7 @@ public class ResponseMergerTest {
}
@Test
public void two_responses_are_merged_reality_dominates_expectations() {
public void two_responses_are_merged_reality_dominates_expectations() throws ExcelImportException {
Excel excel = new Excel("./excel/data/abs_poverty_expect_and_reality_for_merge.xlsx", DATA);
Language lang = Language.ET;
CoordinatedResponses responses = converter.convert(excel, lang);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment