diff --git a/fastexcel-test/src/test/java/cn/idev/excel/test/core/validate/ValidateDemoData.java b/fastexcel-test/src/test/java/cn/idev/excel/test/core/validate/ValidateDemoData.java new file mode 100644 index 000000000..abc528e5e --- /dev/null +++ b/fastexcel-test/src/test/java/cn/idev/excel/test/core/validate/ValidateDemoData.java @@ -0,0 +1,21 @@ +package cn.idev.excel.test.core.validate; + +import cn.idev.excel.annotation.ExcelProperty; +import lombok.Data; + +import java.math.BigDecimal; + +/** + * @author wangmeng + * @since 2025/3/22 + */ +@Data +public class ValidateDemoData { + + @ExcelProperty(value = "订单号", notNull = true) + private String orderNo; + @ExcelProperty(value = "用户名", notNull = true) + private String username; + @ExcelProperty(value = "金额") + private BigDecimal amount; +} diff --git a/fastexcel-test/src/test/java/cn/idev/excel/test/core/validate/ValidateTest.java b/fastexcel-test/src/test/java/cn/idev/excel/test/core/validate/ValidateTest.java new file mode 100644 index 000000000..3599faeb1 --- /dev/null +++ b/fastexcel-test/src/test/java/cn/idev/excel/test/core/validate/ValidateTest.java @@ -0,0 +1,118 @@ +package cn.idev.excel.test.core.validate; + +import cn.idev.excel.FastExcel; +import cn.idev.excel.exception.ExcelDataConvertException; +import cn.idev.excel.read.builder.ExcelReaderBuilder; +import cn.idev.excel.read.listener.PageReadListener; +import cn.idev.excel.read.metadata.holder.ValidateErrorHolder; +import cn.idev.excel.read.processor.FileErrorHandler; +import cn.idev.excel.test.util.TestFileUtil; +import com.alibaba.fastjson2.JSON; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.List; + +/** + * Test cases for reading validation + * + * @author wangmeng + */ +@Slf4j +public class ValidateTest { + + + @Test + public void test() { + // 不存在转换失败情况 + String fileName = TestFileUtil.getPath() + "validate" + File.separator + "checkRead" + ".xlsx"; + testCheckRead(fileName); + testCheckOutFile(fileName); + existConvertException(fileName); + // 存在转换失败 + String fileName2 = TestFileUtil.getPath() + "validate" + File.separator + "checkRead2" + ".xlsx"; + existConvertException(fileName2); + existConvertException(fileName); + // 手动添加业务校验 + testAddError(fileName); + testAddError(fileName2); + } + + + public void testCheckRead(String fileName) { + ExcelReaderBuilder readerBuilder = FastExcel.read(fileName).head(ValidateDemoData.class); + List list = readerBuilder.registerReadListener(new PageReadListener<>(l -> { + System.out.println("读取内容:" + JSON.toJSONString(l)); + })).validate().sheet().doReadSync(); + + ValidateErrorHolder errorHolder = readerBuilder.getErrorHolder(); + System.out.println(JSON.toJSONString(errorHolder)); + String errorText = readerBuilder.handleError(); + System.out.println(errorText); + } + + public void testCheckOutFile(String fileName) { + ExcelReaderBuilder readerBuilder = FastExcel.read(fileName).head(ValidateDemoData.class); + List list = readerBuilder.registerReadListener(new PageReadListener<>(l -> { + System.out.println("读取内容:" + JSON.toJSONString(l)); + })).validate().setErrorHandler(FileErrorHandler.INSTANCE).sheet().doReadSync(); + + ValidateErrorHolder errorHolder = readerBuilder.getErrorHolder(); + System.out.println(JSON.toJSONString(errorHolder)); + File errorFile = readerBuilder.handleError(); + System.out.println("错误信息输出到" + errorFile.getAbsolutePath()); + } + + + public void existConvertException(String fileName) { + ExcelReaderBuilder readerBuilder = FastExcel.read(fileName).head(ValidateDemoData.class); + try { + List list = readerBuilder.registerReadListener(new PageReadListener<>(l -> { + System.out.println("读取内容:" + JSON.toJSONString(l)); + })).validate().setErrorHandler(FileErrorHandler.INSTANCE).sheet().doReadSync(); + } catch (ExcelDataConvertException e) { + log.info("存在类型转换失败异常"); + } + ValidateErrorHolder errorHolder = readerBuilder.getErrorHolder(); + System.out.println(JSON.toJSONString(errorHolder)); + File errorFile = readerBuilder.handleError(); + System.out.println("错误信息输出到" + errorFile.getAbsolutePath()); + } + + + public void testAddError(String fileName) { + ExcelReaderBuilder readerBuilder = FastExcel.read(fileName).head(ValidateDemoData.class); + List list = null; + try { + list = readerBuilder.registerReadListener(new PageReadListener<>(l -> { + System.out.println("读取内容:" + JSON.toJSONString(l)); + })).validate().setErrorHandler(FileErrorHandler.INSTANCE).sheet().doReadSync(); + } catch (ExcelDataConvertException e) { + log.info("存在类型转换失败异常"); + } + ValidateErrorHolder errorHolder = readerBuilder.getErrorHolder(); + System.out.println(JSON.toJSONString(errorHolder)); + + // 手动业务校验 + if(CollectionUtils.isEmpty(list)){ + return; + } + + for (int i = 0; i < list.size(); i++) { + if(true){ + errorHolder.addError(i+1,"订单号已存在"); + } + if(true){ + errorHolder.addError(i+1,"用户不存在"); + } + } + + + File errorFile = readerBuilder.handleError(); + System.out.println("错误信息输出到" + errorFile.getAbsolutePath()); + } + + +} diff --git a/fastexcel-test/src/test/resources/validate/checkRead.xlsx b/fastexcel-test/src/test/resources/validate/checkRead.xlsx new file mode 100644 index 000000000..6040bfec7 Binary files /dev/null and b/fastexcel-test/src/test/resources/validate/checkRead.xlsx differ diff --git a/fastexcel-test/src/test/resources/validate/checkRead2.xlsx b/fastexcel-test/src/test/resources/validate/checkRead2.xlsx new file mode 100644 index 000000000..a9929c5aa Binary files /dev/null and b/fastexcel-test/src/test/resources/validate/checkRead2.xlsx differ diff --git a/fastexcel/src/main/java/cn/idev/excel/annotation/ExcelProperty.java b/fastexcel/src/main/java/cn/idev/excel/annotation/ExcelProperty.java index 31a948601..c58b87f57 100644 --- a/fastexcel/src/main/java/cn/idev/excel/annotation/ExcelProperty.java +++ b/fastexcel/src/main/java/cn/idev/excel/annotation/ExcelProperty.java @@ -65,4 +65,10 @@ */ @Deprecated String format() default ""; + + /** + * use with {@link .ValidateReadListener} to verify whether a field is empty + * @return whether the field can be null + */ + boolean notNull() default false; } diff --git a/fastexcel/src/main/java/cn/idev/excel/read/builder/ExcelReaderBuilder.java b/fastexcel/src/main/java/cn/idev/excel/read/builder/ExcelReaderBuilder.java index 7cbf5908b..e81c2356e 100644 --- a/fastexcel/src/main/java/cn/idev/excel/read/builder/ExcelReaderBuilder.java +++ b/fastexcel/src/main/java/cn/idev/excel/read/builder/ExcelReaderBuilder.java @@ -10,7 +10,11 @@ import cn.idev.excel.event.AnalysisEventListener; import cn.idev.excel.event.SyncReadListener; import cn.idev.excel.read.listener.ModelBuildEventListener; +import cn.idev.excel.read.listener.ValidateReadListener; import cn.idev.excel.read.metadata.ReadWorkbook; +import cn.idev.excel.read.metadata.holder.ValidateErrorHolder; +import cn.idev.excel.read.processor.TextErrorHandler; +import cn.idev.excel.read.processor.ValidateErrorHandler; import cn.idev.excel.support.ExcelTypeEnum; import java.io.File; import java.io.InputStream; @@ -18,6 +22,7 @@ import java.util.HashSet; import java.util.List; import javax.xml.parsers.SAXParserFactory; +import lombok.Getter; /** * Build ExcelReader @@ -30,6 +35,12 @@ public class ExcelReaderBuilder extends AbstractExcelReaderParameterBuilder errorHandler; + + @Getter + private ValidateErrorHolder errorHolder; + public ExcelReaderBuilder() { this.readWorkbook = new ReadWorkbook(); } @@ -282,4 +293,28 @@ public ExcelReaderBuilder ignoreHiddenSheet(Boolean ignoreHiddenSheet) { readWorkbook.setIgnoreHiddenSheet(ignoreHiddenSheet); return this; } + + public ExcelReaderBuilder setErrorHandler(ValidateErrorHandler validateErrorHandler) { + this.errorHandler = validateErrorHandler; + return this; + } + + /** + * enable validate, + * Note: The default {@link TextErrorHandler} is used here. + * If a replacement is needed, call {@link ExcelReaderBuilder#setErrorHandler} after this method. + * + * @return + */ + public ExcelReaderBuilder validate() { + ValidateReadListener validateReadListener = new ValidateReadListener<>(); + registerReadListener(validateReadListener); + errorHolder = validateReadListener; + setErrorHandler(TextErrorHandler.INSTANCE); + return this; + } + + public T handleError() { + return (T) getErrorHandler().handleError(getErrorHolder()); + } } diff --git a/fastexcel/src/main/java/cn/idev/excel/read/listener/ValidateReadListener.java b/fastexcel/src/main/java/cn/idev/excel/read/listener/ValidateReadListener.java new file mode 100644 index 000000000..5ba73cd2f --- /dev/null +++ b/fastexcel/src/main/java/cn/idev/excel/read/listener/ValidateReadListener.java @@ -0,0 +1,137 @@ +package cn.idev.excel.read.listener; + +import cn.idev.excel.annotation.ExcelProperty; +import cn.idev.excel.context.AnalysisContext; +import cn.idev.excel.exception.ExcelDataConvertException; +import cn.idev.excel.metadata.Head; +import cn.idev.excel.metadata.data.ReadCellData; +import cn.idev.excel.read.metadata.ValidateError; +import cn.idev.excel.read.metadata.holder.ValidateErrorHolder; +import cn.idev.excel.read.metadata.property.ExcelReadHeadProperty; +import java.io.File; +import java.lang.reflect.Field; +import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * validate read listener + * + */ +public class ValidateReadListener implements ReadListener, ValidateErrorHolder { + + private static final Logger LOGGER = LoggerFactory.getLogger(ValidateReadListener.class); + + private boolean existValidate = true; + + private final Map fieldMap = new HashMap<>(); + + private final Map> errorMap = new TreeMap<>(); + + private static final String CHECK_EMPTY_TEXT = "%s cannot be null or empty"; + private static final String CONVERSION_FAIL_TEXT = + "the '%s' field type conversion failed, please enter the correct content"; + + private File sourceFile = null; + private Integer sheetNo; + + /** + * Record {@link ExcelDataConvertException} + * + * @param exception + * @param context + * @throws Exception + */ + @Override + public void onException(Exception exception, AnalysisContext context) throws Exception { + if (exception instanceof ExcelDataConvertException) { + ExcelDataConvertException convertException = ((ExcelDataConvertException) exception); + Field field = convertException.getExcelContentProperty().getField(); + String headName = fieldMap.get(field); + ValidateError error = new ValidateError( + context.readRowHolder().getRowIndex(), headName, String.format(CONVERSION_FAIL_TEXT, headName)); + // mark conversion failure + error.setConvertError(true); + addError(error); + } + } + + /** + * Initialize all fields that require validation + * + * @param headMap + * @param context + */ + @Override + public void invokeHead(Map> headMap, AnalysisContext context) { + ExcelReadHeadProperty excelReadHeadProperty = + context.currentReadHolder().excelReadHeadProperty(); + for (Head head : excelReadHeadProperty.getHeadMap().values()) { + Field field = head.getField(); + String headName = String.join("-", head.getHeadNameList()); + field.setAccessible(true); + fieldMap.put(field, headName); + + ExcelProperty excelProperty = field.getAnnotation(ExcelProperty.class); + if (excelProperty != null && excelProperty.notNull()) {} + } + if (fieldMap.isEmpty()) { + existValidate = false; + } + if (sourceFile == null) { + sourceFile = context.readWorkbookHolder().getFile(); + } + sheetNo = context.readSheetHolder().getSheetNo(); + } + + @Override + public void invoke(T data, AnalysisContext context) { + if (existValidate) { + checkNotNull(data, context); + } + } + + private void checkNotNull(T data, AnalysisContext context) { + Integer rowIndex = context.readRowHolder().getRowIndex(); + fieldMap.forEach((field, headName) -> { + try { + Object attribute = field.get(data); + if (attribute == null || (field.getType().equals(String.class) && ((String) attribute).isEmpty())) { + addError(rowIndex, String.format(CHECK_EMPTY_TEXT, headName)); + } + } catch (IllegalAccessException e) { + LOGGER.warn("failed to retrieve field properties through reflection"); + } + }); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) {} + + @Override + public Map> getError() { + return errorMap; + } + + @Override + public void addError(Integer rowNum, String message) { + ValidateError error = new ValidateError(rowNum, message); + errorMap.computeIfAbsent(rowNum, key -> new ArrayList<>()); + errorMap.get(rowNum).add(error); + } + + public void addError(ValidateError error) { + errorMap.computeIfAbsent(error.getRowNum(), key -> new ArrayList<>()); + errorMap.get(error.getRowNum()).add(error); + } + + @Override + public File getSourceFile() { + return sourceFile; + } + + @Override + public Integer getSheetNo() { + return sheetNo; + } +} diff --git a/fastexcel/src/main/java/cn/idev/excel/read/metadata/ValidateError.java b/fastexcel/src/main/java/cn/idev/excel/read/metadata/ValidateError.java new file mode 100644 index 000000000..04debd9ba --- /dev/null +++ b/fastexcel/src/main/java/cn/idev/excel/read/metadata/ValidateError.java @@ -0,0 +1,50 @@ +package cn.idev.excel.read.metadata; + +import cn.idev.excel.exception.ExcelDataConvertException; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * validation error message + * + * @author wangmeng + */ +@Getter +@Setter +@EqualsAndHashCode +@NoArgsConstructor +public class ValidateError { + + /** + * rowNum + */ + private Integer rowNum; + + /** + * headName + */ + private String headName; + + /** + * message + */ + private String message; + + /** + * Record whether it was caused by {@link ExcelDataConvertException} + */ + private boolean convertError = false; + + public ValidateError(Integer rowNum, String headName, String message) { + this.rowNum = rowNum; + this.headName = headName; + this.message = message; + } + + public ValidateError(Integer rowNum, String message) { + this.rowNum = rowNum; + this.message = message; + } +} diff --git a/fastexcel/src/main/java/cn/idev/excel/read/metadata/holder/ValidateErrorHolder.java b/fastexcel/src/main/java/cn/idev/excel/read/metadata/holder/ValidateErrorHolder.java new file mode 100644 index 000000000..0754ec5b4 --- /dev/null +++ b/fastexcel/src/main/java/cn/idev/excel/read/metadata/holder/ValidateErrorHolder.java @@ -0,0 +1,42 @@ +package cn.idev.excel.read.metadata.holder; + +import cn.idev.excel.read.metadata.ValidateError; +import java.io.File; +import java.util.List; +import java.util.Map; + +/** + * validate read listener + * + * @author wangmeng + */ +public interface ValidateErrorHolder { + + /** + * get a map of error messages + * + * @return {@link List }<{@link ValidateError }> + */ + Map> getError(); + + /** + * add an error message + * + * @param rowNum rowNum, + * @param message error message + */ + void addError(Integer rowNum, String message); + + /** + * get the source file for reading + * + * @return file for reading + */ + File getSourceFile(); + + /** + * get sheetNo + * @return sheetNo + */ + Integer getSheetNo(); +} diff --git a/fastexcel/src/main/java/cn/idev/excel/read/processor/FileErrorHandler.java b/fastexcel/src/main/java/cn/idev/excel/read/processor/FileErrorHandler.java new file mode 100644 index 000000000..4acac14ae --- /dev/null +++ b/fastexcel/src/main/java/cn/idev/excel/read/processor/FileErrorHandler.java @@ -0,0 +1,105 @@ +package cn.idev.excel.read.processor; + +import cn.idev.excel.read.metadata.ValidateError; +import cn.idev.excel.read.metadata.holder.ValidateErrorHolder; +import cn.idev.excel.util.FileUtils; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * handle the error message as file + * + */ +public class FileErrorHandler implements ValidateErrorHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(FileErrorHandler.class); + + private static final String COLUMN_NAME = "Error message"; + + public static final FileErrorHandler INSTANCE = new FileErrorHandler(); + + @Override + public File handleError(ValidateErrorHolder errorHolder) { + File sourceFile = errorHolder.getSourceFile(); + try { + File tempFile = FileUtils.createErrorTempleFile(sourceFile); + fillInErrorData(tempFile, errorHolder); + return tempFile; + } catch (IOException e) { + LOGGER.warn("create error temp file fail", e); + } + return null; + } + + /** + * fill error data into the Excel file + * + * @param tempFile + * @param errorHolder + * @throws IOException + */ + public void fillInErrorData(File tempFile, ValidateErrorHolder errorHolder) throws IOException { + Map> errorMap = errorHolder.getError(); + + Workbook workbook = null; + // open the original Excel file and add error messages + try (FileInputStream inputStream = new FileInputStream(tempFile)) { + workbook = new XSSFWorkbook(inputStream); + Sheet sheet = workbook.getSheetAt(errorHolder.getSheetNo()); + + // add error column headers + Row headerRow = sheet.getRow(0); + short lastCellNum = headerRow.getLastCellNum(); + // check if the error column already exists + Cell lastValidCell = headerRow.getCell(lastCellNum - 1); + if (lastValidCell != null) { + if (!COLUMN_NAME.equals(lastValidCell.getStringCellValue())) { + Cell errorHeaderCell = headerRow.createCell(lastCellNum); + errorHeaderCell.setCellValue(COLUMN_NAME); + errorMap.forEach((rowNum, list) -> { + Row row = sheet.getRow(rowNum); + if (row != null) { + Cell errorCell = row.createCell(lastCellNum); + errorCell.setCellValue(mergeText(list)); + } + }); + } else { + int lastRowNum = sheet.getLastRowNum(); + for (int rowNum = 1; rowNum <= lastRowNum; rowNum++) { + Row row = sheet.getRow(rowNum); + String setErrorMsg = mergeText(errorMap.get(rowNum)); + // if there is no error information to set, the old error information should be cleared + Cell errorCell = row.getCell(lastCellNum - 1); + if (setErrorMsg == null) { + if (errorCell != null) { + errorCell.setCellValue((String) null); + } + } else { + if (errorCell == null) { + errorCell = row.createCell(lastCellNum - 1); + } + errorCell.setCellValue(setErrorMsg); + } + } + } + } + } + + try (FileOutputStream outputStream = new FileOutputStream(tempFile)) { + // write it back + workbook.write(outputStream); + workbook.close(); + } + } +} diff --git a/fastexcel/src/main/java/cn/idev/excel/read/processor/TextErrorHandler.java b/fastexcel/src/main/java/cn/idev/excel/read/processor/TextErrorHandler.java new file mode 100644 index 000000000..e52c35445 --- /dev/null +++ b/fastexcel/src/main/java/cn/idev/excel/read/processor/TextErrorHandler.java @@ -0,0 +1,31 @@ +package cn.idev.excel.read.processor; + +import cn.idev.excel.read.metadata.ValidateError; +import cn.idev.excel.read.metadata.holder.ValidateErrorHolder; +import java.util.List; +import java.util.Map; + +/** + * handle the error message as text + * + */ +public class TextErrorHandler implements ValidateErrorHandler { + + private static final String ERROR_STRING = "error message:"; + private static final String LINE_ERROR_STRING = " Line %s: %s"; + + public static final TextErrorHandler INSTANCE = new TextErrorHandler(); + + @Override + public String handleError(ValidateErrorHolder errorHolder) { + Map> errorMap = errorHolder.getError(); + if (errorMap.isEmpty()) { + return null; + } + StringBuilder sb = new StringBuilder(ERROR_STRING); + errorMap.forEach((index, errorList) -> { + sb.append(String.format(LINE_ERROR_STRING, index, mergeText(errorList))); + }); + return sb.toString(); + } +} diff --git a/fastexcel/src/main/java/cn/idev/excel/read/processor/ValidateErrorHandler.java b/fastexcel/src/main/java/cn/idev/excel/read/processor/ValidateErrorHandler.java new file mode 100644 index 000000000..8b0d5eb3a --- /dev/null +++ b/fastexcel/src/main/java/cn/idev/excel/read/processor/ValidateErrorHandler.java @@ -0,0 +1,21 @@ +package cn.idev.excel.read.processor; + +import cn.idev.excel.read.metadata.ValidateError; +import cn.idev.excel.read.metadata.holder.ValidateErrorHolder; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Handle error messages + * the source of the error is + * {@link cn.idev.excel.read.metadata.holder.ValidateErrorHolder} + * + */ +public interface ValidateErrorHandler { + + T handleError(ValidateErrorHolder errorHolder); + + default String mergeText(List list) { + return list.stream().map(each -> each.getMessage() + ";").collect(Collectors.joining("")); + } +} diff --git a/fastexcel/src/main/java/cn/idev/excel/util/FileUtils.java b/fastexcel/src/main/java/cn/idev/excel/util/FileUtils.java index ed5e626f6..836ff06d8 100644 --- a/fastexcel/src/main/java/cn/idev/excel/util/FileUtils.java +++ b/fastexcel/src/main/java/cn/idev/excel/util/FileUtils.java @@ -28,12 +28,15 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; import java.util.UUID; import org.apache.poi.util.TempFile; public class FileUtils { public static final String POI_FILES = "poifiles"; public static final String EX_CACHE = "excache"; + + public static final String ERROR_FILES = "errorfiles"; /** * If a server has multiple projects in use at the same time, a directory with the same name will be created under * the temporary directory, but each project is run by a different user, so there is a permission problem, so each @@ -52,6 +55,11 @@ public class FileUtils { */ private static String cachePath = tempFilePrefix + EX_CACHE + File.separator; + /** + * Used to store error temporary files + */ + private static String errorFilePath = tempFilePrefix + ERROR_FILES + File.separator; + private static final int WRITE_BUFF_SIZE = 8192; private FileUtils() {} @@ -64,6 +72,10 @@ private FileUtils() {} // Initialize the cache directory File cacheFile = new File(cachePath); createDirectory(cacheFile); + // Initialize the error file directory + File errorFile = new File(errorFilePath); + createDirectory(errorFile); + errorFile.deleteOnExit(); } /** @@ -202,6 +214,16 @@ public static void delete(File file) { } } + /** + * Generate a temporary error file based on the source file + * @param sourceFile sourceFile + */ + public static File createErrorTempleFile(File sourceFile) throws IOException { + File tempFile = new File(errorFilePath, UUID.randomUUID() + ".xlsx"); + writeToFile(tempFile, Files.newInputStream(sourceFile.toPath())); + return tempFile; + } + public static String getTempFilePrefix() { return tempFilePrefix; }