Merge pull request #290 from pdowler/master

caom2-repo: add default read/connection timeouts and set methods to override

caom2-repo/build.gradle

@@ -17,7 +17,7 @@ sourceCompatibility = 1.8
 
 group = 'org.opencadc'
 
-version = '1.4.5'
+version = '1.4.6'
 
 description = 'OpenCADC CAOM repository client library'
 def git_url = 'https://github.com/opencadc/caom2db'

caom2-repo/src/main/java/ca/nrc/cadc/caom2/repo/client/RepoClient.java

@@ -3,7 +3,7 @@
  *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
  **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
  *
- *  (c) 2018.                            (c) 2018.
+ *  (c) 2023.                            (c) 2023.
  *  Government of Canada                 Gouvernement du Canada
  *  National Research Council            Conseil national de recherches
  *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -139,6 +139,9 @@ public class RepoClient {
 
     private boolean isObsAvailable = false;
     private boolean isDelAvailable = false;
+
+    private int connectionTimeout = 6000;  // default: 6 sec
+    private int  readTimeout = 60000;      // default: 60 sec
 
     /**
      * @return the isObsAvailable
@@ -267,6 +270,25 @@ private void init() {
         this.isDelAvailable = true;
     }
 
+    /**
+     * Override the default connection timeout (6000ms aka 6sec).
+     * 
+     * @param connectionTimeout new timeout value
+     */
+    public void setConnectionTimeout(int connectionTimeout) {
+        this.connectionTimeout = connectionTimeout;
+    }
+
+    /**
+     * Override the default read timeout (60000ms aka 60sec).
+     * 
+     * @param readTimeout new timeout value
+     */
+    public void setReadTimeout(int readTimeout) {
+        this.readTimeout = readTimeout;
+    }
+
+
     public List<DeletedObservation> getDeleted(String collection, Date start, Date end, Integer maxrec) {
         return readDeletedEntityList(new DeletionListReader(), collection, start, end, maxrec);
         // TODO: make call(s) to the deletion endpoint until requested number of
@@ -469,6 +491,8 @@ private List<ObservationState> readObservationStateList(ObservationStateListRead
             }
 
             HttpGet get = new HttpGet(url, true);
+            get.setConnectionTimeout(connectionTimeout);
+            get.setReadTimeout(readTimeout);
             try {
                 get.prepare();
             } catch (AccessControlException | NotAuthenticatedException e) {
@@ -591,7 +615,8 @@ private List<DeletedObservation> readDeletedEntityList(DeletionListReader transf
             }
 
             HttpGet get = new HttpGet(url, true);
-
+            get.setConnectionTimeout(connectionTimeout);
+            get.setReadTimeout(readTimeout);
             try {
                 get.prepare();
             } catch (AccessControlException | NotAuthenticatedException e) {

icewind/VERSION

@@ -1,6 +1,6 @@
 ## deployable containers have a semantic and build tag
 # semantic version tag: major.minor[.patch]
 # build version tag: timestamp
-VER=0.9.5
+VER=0.9.6
 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")"
 unset VER

icewind/build.gradle

@@ -20,7 +20,7 @@ dependencies {
     implementation 'org.opencadc:cadc-util:[1.6,2.0)'
     implementation 'org.opencadc:caom2:[2.4.4,2.5)'
     implementation 'org.opencadc:caom2persistence:[2.4.14,2.5)'
-    implementation 'org.opencadc:caom2-repo:[1.4.5,1.5)'
+    implementation 'org.opencadc:caom2-repo:[1.4.6,1.5)'
 
     // needed for validation
     implementation 'org.opencadc:caom2-compute:[2.4.6,2.5)'

icewind/src/main/java/org/opencadc/icewind/DeletionHarvester.java

@@ -138,7 +138,10 @@ public void setInitHarvestState(boolean initHarvestState) {
     private void init() {
         // source
         this.repoClient = new RepoClient(src.getResourceID(), 1);
-
+        // TODO: make these configurable
+        repoClient.setConnectionTimeout(18000); // 18 sec
+        repoClient.setReadTimeout(120000);      // 2 min
+
         // destination
         final String destDS = "jdbc/DeletionHarvester";
 

icewind/src/main/java/org/opencadc/icewind/ObservationHarvester.java

@@ -115,12 +115,11 @@ public class ObservationHarvester extends Harvester {
 
     private final URI basePublisherID;
     private final boolean nochecksum;
-    private RepoClient srcObservationService;
+    private RepoClient srcRepoClient;
     private ObservationDAO srcObservationDAO;
     private ObservationDAO destObservationDAO;
     private HarvestSkipURIDAO harvestSkipDAO;
     private boolean skipped;
-    private boolean computePlaneMetadata;
     private boolean ready = false;
     private int ingested = 0;
 
@@ -129,7 +128,6 @@ public ObservationHarvester(HarvesterResource src, HarvesterResource dest, Strin
         super(Observation.class, src, dest, collection, batchSize, full);
         this.basePublisherID = basePublisherID;
         this.nochecksum = nochecksum;
-        this.computePlaneMetadata = false;
         init(nthreads);
     }
 
@@ -142,8 +140,11 @@ public int getIngested() {
     }
 
     private void init(int nthreads) {
-        this.srcObservationService = new RepoClient(src.getResourceID(), nthreads);
-
+        this.srcRepoClient = new RepoClient(src.getResourceID(), nthreads);
+        // TODO: make these configurable
+        srcRepoClient.setConnectionTimeout(18000); // 18 sec
+        srcRepoClient.setReadTimeout(120000);      // 2 min
+
         // dest is always a database
         final String destDS = "jdbc/ObservationHarvester";
         Map<String, Object> destConfig = getConfigDAO(dest);
@@ -177,10 +178,10 @@ private void init(int nthreads) {
         }
         initHarvestState(destObservationDAO.getDataSource(), Observation.class);
 
-        if (srcObservationService.isObsAvailable()) {
+        if (srcRepoClient.isObsAvailable()) {
             ready = true;
         } else {
-            log.error("Not available: obs endpoint in " + srcObservationService.toString());
+            log.error("Not available: obs endpoint in " + srcRepoClient.toString());
         }
     }
 
@@ -290,7 +291,7 @@ private Progress doit() {
                 if (srcObservationDAO != null) {
                     obsList = srcObservationDAO.getList(collection, startDate, endDate, batchSize + 1);
                 } else {
-                    obsList = srcObservationService.getList(collection, startDate, endDate, batchSize + 1);
+                    obsList = srcRepoClient.getList(collection, startDate, endDate, batchSize + 1);
                 }
                 entityList = wrap(obsList);
             }
@@ -689,7 +690,7 @@ private List<SkippedWrapperURI<ObservationResponse>> getSkipped(Date start) thro
                 log.debug("getSkipped: " + hs.getSkipID());
                 listUris.add(new ObservationURI(hs.getSkipID()));
             }
-            List<ObservationResponse> listResponses = srcObservationService.get(listUris);
+            List<ObservationResponse> listResponses = srcRepoClient.get(listUris);
             log.warn("getSkipped: " + skip.size() + " HarvestSkipURI -> " + listResponses.size() + " ObservationResponse");
 
             for (ObservationResponse o : listResponses) {