Fix DAP4 remotetest server

Warning: This PR is a follow on to PR #2555 and should not be merged until that prior PR has been merged. The changeset for this PR is a delta on the PR #2555.

This PR re-enables the use of the server *remotetest.unidata.ucar.edu/d4ts*
to test several features:
1. Show that access over the Internet to servers using the DAP4 protocol works.
2. Test that DAP4 support in the [Thredds Data Server](https://github.com/Unidata/tds) is operating correctly.
4. Test that the DAP4 support in the [netcdf-java library](https://github.com/Unidata/netcdf-java) library and the DAP4 support in the netcdf-c library are consistent and are interoperable.

The test inputs (primarily *\*.nc* files) provided in the netcdf-c library
are also used by the DAP4 Test Server (aka d4ts) to present web access to a
collection of data files accessible via the DAP4 protocol and which can be
used for testing Internet access to a working server.

To be precise, this version of d4ts is currently in unmerged branches
of the *netcdf-java* and *tds* Github repositories and so are not actually
in the main repositories *yet*. However, the *d4ts.war* file was created
from that branch and used to populate the *remotetest.unidata.ucar.edu*
server

The two other remote servers that were used in the past are *Hyrax* (OPenDAP.org)
and *thredds-test*. These will continue to remain disabled until
those servers can be fixed.

## Primary Changes

* Rebuild the *baselineremote* directory. This directory contains the validation data needed to test the remote servers.
* Re-enable using remotetest.unidata.ucar.edu as part of the DAP4 testing process.
* Fix the *dap4_test/test_remote.sh* test script to match the current available test data.
* Make some changes to libdap4 to improve the ability to catch malformed data streams [affects a lot of files in libdap4].

## Misc. Unrelated Changes

* Remove a raft of warnings, especially in nc_test4/tst_quantize.c.
* Add some additional explanatory information to the NCZarr documentation.
* Cleanup some Doxygen errors in the docs file and reorder some files.

RELEASE_NOTES.md

@@ -16,7 +16,9 @@ This file contains a high-level description of this package's evolution. Release
 
 #### Changes
 
-* [Bug Fix] Make major changes to libdap4 and dap4_test to update the non-remote DAP4 tests. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
+* [Bug Fix] Get remotetest server working for DAP4 and re-enable its use in the netcdf-c library for testing. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
+* [Bug Fix] Fix a race condition when testing missing filters. See [Github #2557](https://github.com/Unidata/netcdf-c/pull/2557). 
+* [Bug Fix] Make major changes to libdap4 and dap4_test to update the non-remote DAP4 tests. See [Github #2555](https://github.com/Unidata/netcdf-c/pull/2555).
 * [Bug Fix] Fix some race conditions due to use of a common file in multiple shell scripts . See [Github #2552](https://github.com/Unidata/netcdf-c/pull/2552).
 * [Enhancement][Documentation] Add Plugins Quick Start Guide.  See [GitHub #2524](https://github.com/Unidata/netcdf-c/pull/2524) for more information.
 * [Enhancement] Add new entries in `netcdf_meta.h`, `NC_HAS_BLOSC` and `NC_HAS_BZ2`. See [Github #2511](https://github.com/Unidata/netcdf-c/issues/2511) and [Github #2512](https://github.com/Unidata/netcdf-c/issues/2512) for more information.

configure.ac

@@ -614,21 +614,14 @@ AC_MSG_CHECKING([whether dap use of remotetest server should be enabled])
 AC_ARG_ENABLE([dap-remote-tests],
               [AS_HELP_STRING([--disable-dap-remote-tests],
                                  [disable dap remote tests])])
-# Default off
-test "x$enable_dap_remote_tests" = xyes || enable_dap_remote_tests=no
+# Default on
+test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes
 if test "x$enable_dap" = "xno" ; then
   enable_dap_remote_tests=no
 fi
 AC_MSG_RESULT($enable_dap_remote_tests)
 
-AC_MSG_CHECKING([whether dap use of remotetest server should be enabled])
-AC_ARG_ENABLE([dap-remote-tests],
-              [AS_HELP_STRING([--disable-dap-remote-tests],
-                                 [disable dap remote tests])])
-test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes
-AC_MSG_RESULT($enable_dap_remote_tests)
-
-AC_MSG_CHECKING([whether use of external servers should be enabled])
+AC_MSG_CHECKING([whether use of external (non-unidata) servers should be enabled])
 AC_ARG_ENABLE([external-server-tests],
               [AS_HELP_STRING([--enable-external-server-tests (default off)],
                                  [enable external server tests])])
@@ -667,11 +660,12 @@ if test "x$REMOTETESTSERVERS" = xno ; then
   dfaltsvc="remotetest.unidata.ucar.edu"
   REMOTETESTSERVERS="${dfaltsvc}"
 fi
-AC_MSG_RESULT([$svclist])
-AC_DEFINE_UNQUOTED([REMOTETESTSERVERS], ["$REMOTETESTSERVERS"], [the testservers for remote tests.])
+msg="${REMOTETESTSERVERS}"
 if test "x$dfaltsvc" != x ; then
-  AC_MSG_NOTICE([Using default: --with-testservers=${dfaltsvc}])
+  msg="${msg} (default)"
 fi
+AC_MSG_RESULT([${msg}])
+AC_DEFINE_UNQUOTED([REMOTETESTSERVERS], ["$REMOTETESTSERVERS"], [the testservers for remote tests.])
 
 # Set the config.h flags
 if test "x$enable_dap" = xyes; then

dap4_test/CMakeLists.txt

@@ -51,25 +51,23 @@ ENDIF()
   ENDIF(BUILD_UTILITIES)
 
   IF(ENABLE_DAP_REMOTE_TESTS)
-#      add_sh_test(dap4_test test_remote)
+      add_sh_test(dap4_test test_remote)
 #      add_sh_test(dap4_test test_hyrax)
 #      add_sh_test(dap4_test test_thredds)
   ENDIF(ENABLE_DAP_REMOTE_TESTS)
 
 ENDIF(ENABLE_TESTS)
 
-FILE(COPY ./baselineraw DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
+#FILE(COPY ./baselineraw DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
 #FILE(COPY ./baseline DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
 #FILE(COPY ./cdltestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
-#FILE(COPY ./daptestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
-#FILE(COPY ./dmrtestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
+#FILE(COPY ./rawtestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
 
 ## Specify files to be distributed by 'make dist'
 FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.c ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.sh
-#${CMAKE_CURRENT_SOURCE_DIR}/daptestfiles
-#${CMAKE_CURRENT_SOURCE_DIR}/dmrtestfiles
-#${CMAKE_CURRENT_SOURCE_DIR}/cdltestfiles
-#${CMAKE_CURRENT_SOURCE_DIR}/baseline
+${CMAKE_CURRENT_SOURCE_DIR}/cdltestfiles
+${CMAKE_CURRENT_SOURCE_DIR}/rawtestfiles
+${CMAKE_CURRENT_SOURCE_DIR}/baseline
 ${CMAKE_CURRENT_SOURCE_DIR}/baselineraw
 )
 SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am)

dap4_test/Makefile.am

@@ -43,16 +43,17 @@ pingurl4_SOURCES = pingurl4.c
 # Disable Dap4 Remote Tests until the test server is working
 if BUILD_UTILITIES
 if ENABLE_DAP_REMOTE_TESTS
+TESTS += test_remote.sh
 # Test require help at the server end
-#  TESTS += test_remote.sh test_hyrax.sh test_thredds.sh
+#  TESTS += test_hyrax.sh test_thredds.sh
 endif
 endif
 
 EXTRA_DIST = CMakeLists.txt test_common.h \
 	d4test_common.sh \
 	test_curlopt.sh test_data.sh test_hyrax.sh test_meta.sh \
 	test_parse.sh test_raw.sh test_remote.sh test_thredds.sh \
-	cdltestfiles rawtestfiles nctestfiles \
+	cdltestfiles rawtestfiles \
 	baseline baselineraw baselineremote baselinehyrax baselinethredds
 
 CLEANFILES = *.exe

dap4_test/baselineremote/test_atomic_array.nc.ncdump

@@ -0,0 +1,48 @@
+netcdf test_atomic_array {
+types:
+  byte enum cloud_class_t {Clear = 0, Cumulonimbus = 1, Stratus = 2, 
+      Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, 
+      Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, 
+      Missing = 127} ;
+  opaque(16) opaque16_t ;
+dimensions:
+	d1 = 1 ;
+	d2 = 2 ;
+	d3 = 3 ;
+	d4 = 4 ;
+	d5 = 5 ;
+variables:
+	ubyte vu8(d2, d3) ;
+	short v16(d4) ;
+	uint vu32(d2, d3) ;
+	double vd(d2) ;
+	char vc(d2) ;
+	string vs(d2, d2) ;
+	opaque16_t vo(d1, d2) ;
+	cloud_class_t primary_cloud(d5) ;
+		cloud_class_t primary_cloud:_FillValue = Missing ;
+data:
+
+ vu8 =
+  255, 1, 2,
+  3, 4, 5 ;
+
+ v16 = 1, 2, 3, 4 ;
+
+ vu32 =
+  5, 4, 3,
+  2, 1, 0 ;
+
+ vd = 17.9, 1024.8 ;
+
+ vc = "@&" ;
+
+ vs =
+  "hello\tworld", "\r\n",
+  "Καλημέα", "abc" ;
+
+ vo =
+  0X0123456789ABCDEF0000000000000000, 0XABCDEF00000000000000000000000000 ;
+
+ primary_cloud = Clear, Stratus, Clear, Cumulonimbus, _ ;
+}

dap4_test/baselineremote/test_atomic_types.nc.ncdump

@@ -0,0 +1,57 @@
+netcdf test_atomic_types {
+types:
+  byte enum cloud_class_t {Clear = 0, Cumulonimbus = 1, Stratus = 2, 
+      Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, 
+      Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, 
+      Missing = 127} ;
+  opaque(16) opaque16_t ;
+variables:
+	byte v8 ;
+	ubyte vu8 ;
+	short v16 ;
+	ushort vu16 ;
+	int v32 ;
+	uint vu32 ;
+	int64 v64 ;
+	uint64 vu64 ;
+	float vf ;
+	double vd ;
+	char vc ;
+	string vs ;
+	opaque16_t vo ;
+	cloud_class_t primary_cloud ;
+		cloud_class_t primary_cloud:_FillValue = Missing ;
+	cloud_class_t secondary_cloud ;
+		cloud_class_t secondary_cloud:_FillValue = Missing ;
+data:
+
+ v8 = -128 ;
+
+ vu8 = 255 ;
+
+ v16 = -32768 ;
+
+ vu16 = _ ;
+
+ v32 = 2147483647 ;
+
+ vu32 = _ ;
+
+ v64 = 9223372036854775807 ;
+
+ vu64 = 18446744073709551615 ;
+
+ vf = 3.141593 ;
+
+ vd = 3.14159265358979 ;
+
+ vc = "@" ;
+
+ vs = "hello\tworld" ;
+
+ vo = 0X0123456789ABCDEF0000000000000000 ;
+
+ primary_cloud = Stratus ;
+
+ secondary_cloud = _ ;
+}

dap4_test/baselineremote/test_enum_1.nc.ncdump

@@ -0,0 +1,13 @@
+netcdf test_enum_1 {
+types:
+  byte enum cloud_class_t {Clear = 0, Cumulonimbus = 1, Stratus = 2, 
+      Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, 
+      Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, 
+      Missing = 127} ;
+variables:
+	cloud_class_t primary_cloud ;
+		cloud_class_t primary_cloud:_FillValue = Missing ;
+data:
+
+ primary_cloud = Stratus ;
+}

dap4_test/baselineremote/test_enum_2.nc.ncdump

@@ -0,0 +1,16 @@
+netcdf test_enum_2 {
+types:
+  byte enum cloud_class_t {Clear = 0, Cumulonimbus = 1, Stratus = 2, 
+      Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, 
+      Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, 
+      Missing = 127} ;
+
+group: h {
+  variables:
+  	cloud_class_t primary_cloud ;
+  		cloud_class_t primary_cloud:_FillValue = Missing ;
+  data:
+
+   primary_cloud = Stratus ;
+  } // group h
+}

dap4_test/baselineremote/test_enum_3.nc.ncdump

@@ -0,0 +1,12 @@
+netcdf test_enum_3 {
+types:
+  byte enum cloud_class_t {Clear = 0, Stratus = 1, Missing = 127} ;
+
+group: h {
+  variables:
+  	cloud_class_t primary_cloud ;
+  data:
+
+   primary_cloud = Stratus ;
+  } // group h
+}

dap4_test/baselineremote/test_enum_array.nc.ncdump

@@ -0,0 +1,15 @@
+netcdf test_enum_array {
+types:
+  byte enum cloud_class_t {Clear = 0, Cumulonimbus = 1, Stratus = 2, 
+      Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, 
+      Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, 
+      Missing = 127} ;
+dimensions:
+	d5 = 5 ;
+variables:
+	cloud_class_t primary_cloud(d5) ;
+		cloud_class_t primary_cloud:_FillValue = Missing ;
+data:
+
+ primary_cloud = Clear, Stratus, Clear, Cumulonimbus, _ ;
+}

dap4_test/baselineremote/test_fill.nc.ncdump

@@ -0,0 +1,14 @@
+netcdf test_fill {
+variables:
+	ubyte uv8 ;
+	short v16 ;
+	int uv32 ;
+		int uv32:_FillValue = 17 ;
+data:
+
+ uv8 = 240 ;
+
+ v16 = 32700 ;
+
+ uv32 = 111000 ;
+}

dap4_test/baselineremote/test_fill_2.nc.ncdump

@@ -0,0 +1,27 @@
+netcdf test_fill_2 {
+types:
+  byte enum cloud_class_t {Clear = 0, Cumulonimbus = 1, Stratus = 2, 
+      Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6, 
+      Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, 
+      Missing = 127} ;
+dimensions:
+	d2 = 2 ;
+variables:
+	cloud_class_t enumvar(d2) ;
+		cloud_class_t enumvar:_FillValue = Missing ;
+	int uv32(d2) ;
+		int uv32:_FillValue = 17 ;
+	ubyte uv8(d2) ;
+		ubyte uv8:_FillValue = 120UB ;
+	short v16(d2) ;
+		short v16:_FillValue = -37s ;
+data:
+
+ enumvar = Stratus, _ ;
+
+ uv32 = 111000, _ ;
+
+ uv8 = 240, _ ;
+
+ v16 = 32700, _ ;
+}

dap4_test/baselineremote/test_groups1.nc.ncdump

@@ -0,0 +1,35 @@
+netcdf test_groups1 {
+dimensions:
+	dim1 = 5 ;
+
+group: g {
+  dimensions:
+  	dim2 = 3 ;
+
+  group: h {
+    dimensions:
+    	dim3 = 7 ;
+    variables:
+    	int v1(dim1) ;
+    	float v2(dim2) ;
+    data:
+
+     v1 = -876354855, -1761252264, 1723477387, -46827465, 1475147969 ;
+
+     v2 = 12, -100, _ ;
+    } // group h
+
+  group: i {
+    dimensions:
+    	dim3 = 7 ;
+    variables:
+    	int v1(dim1) ;
+    	float v3(dim3) ;
+    data:
+
+     v1 = 2, 3, 5, 7, 11 ;
+
+     v3 = 23, 29, 19, 31, 17, 37, 13 ;
+    } // group i
+  } // group g
+}

dap4_test/baselineremote/test_misc1.nc.ncdump

@@ -0,0 +1,24 @@
+netcdf test_misc1 {
+dimensions:
+	lon = 4 ;
+	time = UNLIMITED ; // (0 currently)
+	lat = 6 ;
+	unlim = UNLIMITED ; // (3 currently)
+variables:
+	float var(unlim) ;
+		var:_ChunkSizes = 1024U ;
+	float lon(lon) ;
+		string lon:units = "degrees_east" ;
+	float lat(lat) ;
+		string lat:units = "degrees_north" ;
+
+// global attributes:
+		string :title = "example for workshop" ;
+data:
+
+ var = 0, 1, 3 ;
+
+ lon = _, _, _, _ ;
+
+ lat = _, _, _, _, _, _ ;
+}

dap4_test/baselineremote/test_one_var.nc.ncdump

@@ -0,0 +1,7 @@
+netcdf test_one_var {
+variables:
+	int t ;
+data:
+
+ t = 17 ;
+}

dap4_test/baselineremote/test_one_vararray.nc.ncdump

@@ -0,0 +1,9 @@
+netcdf test_one_vararray {
+dimensions:
+	d2 = 2 ;
+variables:
+	int t(d2) ;
+data:
+
+ t = 17, 37 ;
+}