Merge branch 'datatable-upload-issue' of https://github.com/biodiv-pl…

…atform/biodiv-observation into staging
biodiv-platform · Nov 4, 2024 · 45ccaa9 · 45ccaa9
2 parents aa6970c + 2bbf218
commit 45ccaa9
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 3 deletions.
diff --git a/src/main/java/com/strandls/observation/service/Impl/ObservationDataTableServiceImpl.java b/src/main/java/com/strandls/observation/service/Impl/ObservationDataTableServiceImpl.java
@@ -192,11 +192,14 @@ public Long observationBulkUpload(HttpServletRequest request, ObservationBulkDTO
 						.getAllFilePathsByUser(filesDto).entrySet().stream()
 						.collect(Collectors.toMap(Map.Entry::getKey, e -> (String) e.getValue()));
 
+				String batchSize = properties.getProperty("datatableBatchSize");
+
 				ObservationBulkUploadThread uploadThread = new ObservationBulkUploadThread(observationBulkData, request,
 						observationDao, observationBulkMapperHelper, esUpdate, userService, dataTable,
 						observationBulkData.getContributors(), observationImpl.getAllSpeciesGroup(), traitsList,
 						userGroupIbpList, licenseList, workbook, myImageUpload, resourceService, fileUploadApi,
-						dataTableService, tokenGenerator, observationBulkData.getUserGroup(), headers);
+						dataTableService, tokenGenerator, observationBulkData.getUserGroup(), headers,
+						Long.valueOf(batchSize));
 				Thread thread = new Thread(uploadThread);
 				thread.start();
 			}

diff --git a/src/main/java/com/strandls/observation/util/ObservationBulkUploadThread.java b/src/main/java/com/strandls/observation/util/ObservationBulkUploadThread.java
@@ -62,14 +62,15 @@ public class ObservationBulkUploadThread implements Runnable {
 	private final XSSFWorkbook workbook;
 	private final Map<String, String> myImageUpload;
 	private final TokenGenerator tokenGenerator;
+	private final Long batchSize;
 
 	public ObservationBulkUploadThread(ObservationBulkDTO observationBulkData, HttpServletRequest request,
 			ObservationDAO observationDao, ObservationBulkMapperHelper observationBulkMapperHelper, ESUpdate esUpdate,
 			UserServiceApi userService, DataTableWkt dataTable, Long userId, List<SpeciesGroup> speciesGroupList,
 			List<TraitsValuePair> traitsList, List<UserGroupIbp> userGroupIbpList, List<License> licenseList,
 			XSSFWorkbook workbook, Map<String, String> myImageUpload, ResourceServicesApi resourceService,
 			UploadApi fileUploadApi, DataTableServiceApi dataTableService, TokenGenerator tokenGenerator,
-			String userGroup, Headers headers) {
+			String userGroup, Headers headers, Long batchSize) {
 		super();
 		this.observationBulkData = observationBulkData;
 		this.observationDao = observationDao;
@@ -92,6 +93,7 @@ public ObservationBulkUploadThread(ObservationBulkDTO observationBulkData, HttpS
 		this.tokenGenerator = tokenGenerator;
 		this.userGroup = userGroup;
 		this.myImageUpload = myImageUpload;
+		this.batchSize = batchSize;
 	}
 
 	public void run() {
@@ -124,7 +126,7 @@ public void run() {
 					observationIds.add(obsId);
 				}
 
-				if (observationIds.size() >= 200) {
+				if (observationIds.size() >= batchSize) {
 					String observationList = StringUtils.join(observationIds, ',');
 					ESBulkUploadThread updateThread = new ESBulkUploadThread(esUpdate, observationList);
 					Thread thread = new Thread(updateThread);

diff --git a/src/main/resources/config.properties b/src/main/resources/config.properties
@@ -29,3 +29,4 @@ classificationId=1
 locationinfo_layer_view=lyr115_location_information
 time_zone=Asia/Kolkata
 pull_function=null_pull_function
+datatableBatchSize=200