Link to LibHTP as crate (#11)

Co-authored-by: [email protected] <[email protected]@vid-mdedprd3067.chimera.cyber.gc.ca>
cccs-rtmorti · Oct 24, 2022 · e7075f5 · e7075f5
1 parent f0deef6
commit e7075f5
Show file tree

Hide file tree

Showing 10 changed files with 17 additions and 140 deletions.
diff --git a/configure.ac b/configure.ac
@@ -1447,101 +1447,6 @@
         [test "x$install_suricata_update" = "xyes"])
     AC_SUBST([install_suricata_update_reason])
 
-  # libhtp
-    AC_ARG_ENABLE(non-bundled-htp,
-           AS_HELP_STRING([--enable-non-bundled-htp], [Enable the use of an already installed version of htp]),[enable_non_bundled_htp=$enableval],[enable_non_bundled_htp=no])
-    AS_IF([test "x$enable_non_bundled_htp" = "xyes"], [
-        PKG_CHECK_MODULES([libhtp], htp,, [with_pkgconfig_htp=no])
-        if test "$with_pkgconfig_htp" != "no"; then
-            CPPFLAGS="${CPPFLAGS} ${libhtp_CFLAGS}"
-            LIBS="${LIBS} ${libhtp_LIBS}"
-        fi
-
-        AC_ARG_WITH(libhtp_includes,
-                [  --with-libhtp-includes=DIR  libhtp include directory],
-                [with_libhtp_includes="$withval"],[with_libhtp_includes=no])
-        AC_ARG_WITH(libhtp_libraries,
-                [  --with-libhtp-libraries=DIR    libhtp library directory],
-                [with_libhtp_libraries="$withval"],[with_libhtp_libraries="no"])
-
-        if test "$with_libhtp_includes" != "no"; then
-            CPPFLAGS="-I${with_libhtp_includes} ${CPPFLAGS}"
-        fi
-
-        if test "$with_libhtp_libraries" != "no"; then
-            LDFLAGS="${LDFLAGS} -L${with_libhtp_libraries}"
-        fi
-
-        AC_CHECK_HEADER(htp/htp.h,,[AC_MSG_ERROR(htp/htp.h not found ...)])
-
-        LIBHTP=""
-        AC_CHECK_LIB(htp, htp_conn_create,, LIBHTP="no")
-        if test "$LIBHTP" = "no"; then
-            echo
-            echo "   ERROR! libhtp library not found"
-            echo
-            exit 1
-        fi
-        PKG_CHECK_MODULES(LIBHTPMINVERSION, [htp >= 0.5.40],[libhtp_minver_found="yes"],[libhtp_minver_found="no"])
-        if test "$libhtp_minver_found" = "no"; then
-            PKG_CHECK_MODULES(LIBHTPDEVVERSION, [htp = 0.5.X],[libhtp_devver_found="yes"],[libhtp_devver_found="no"])
-            if test "$libhtp_devver_found" = "no"; then
-                echo
-                echo "   ERROR! libhtp was found but it is neither >= 0.5.40, nor the dev 0.5.X"
-                echo
-                exit 1
-            fi
-        fi
-
-        AC_CHECK_LIB([htp], [htp_config_register_request_uri_normalize],AC_DEFINE_UNQUOTED([HAVE_HTP_URI_NORMALIZE_HOOK],[1],[Found htp_config_register_request_uri_normalize function in libhtp]) ,,[-lhtp])
-        # check for htp_tx_get_response_headers_raw
-        AC_CHECK_LIB([htp], [htp_tx_get_response_headers_raw],AC_DEFINE_UNQUOTED([HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW],[1],[Found htp_tx_get_response_headers_raw in libhtp]) ,,[-lhtp])
-        AC_CHECK_LIB([htp], [htp_decode_query_inplace],AC_DEFINE_UNQUOTED([HAVE_HTP_DECODE_QUERY_INPLACE],[1],[Found htp_decode_query_inplace function in libhtp]) ,,[-lhtp])
-        AC_CHECK_LIB([htp], [htp_config_set_decompression_layer_limit],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT],[1],[Found htp_config_set_decompression_layer_limit function in libhtp]) ,,[-lhtp])
-        AC_EGREP_HEADER(htp_config_set_path_decode_u_encoding, htp/htp.h, AC_DEFINE_UNQUOTED([HAVE_HTP_SET_PATH_DECODE_U_ENCODING],[1],[Found usable htp_config_set_path_decode_u_encoding function in libhtp]) )
-        AC_CHECK_LIB([htp], [htp_config_set_lzma_memlimit],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT],[1],[Found htp_config_set_lzma_memlimit function in libhtp]) ,,[-lhtp])
-        AC_CHECK_LIB([htp], [htp_config_set_lzma_layers],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_LZMA_LAYERS],[1],[Found htp_config_set_lzma_layers function in libhtp]) ,,[-lhtp])
-        AC_CHECK_LIB([htp], [htp_config_set_compression_bomb_limit],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT],[1],[Found htp_config_set_compression_bomb_limit function in libhtp]) ,,[-lhtp])
-        AC_CHECK_LIB([htp], [htp_config_set_compression_time_limit],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT],[1],[Found htp_config_set_compression_time_limit function in libhtp]) ,,[-lhtp])
-    ])
-
-    if test "x$enable_non_bundled_htp" = "xno"; then
-        # test if we have a bundled htp
-        if test -d "$srcdir/libhtp"; then
-            AC_CONFIG_SUBDIRS([libhtp])
-            HTP_DIR="libhtp"
-            AC_SUBST(HTP_DIR)
-            HTP_LDADD="../libhtp/htp/libhtp.la"
-            AC_SUBST(HTP_LDADD)
-            # make sure libhtp is added to the includes
-            CPPFLAGS="-I\${srcdir}/../libhtp/ ${CPPFLAGS}"
-
-            AC_CHECK_HEADER(iconv.h,,[AC_MSG_ERROR(iconv.h not found ...)])
-            AC_CHECK_LIB(iconv, libiconv_close)
-            AC_DEFINE_UNQUOTED([HAVE_HTP_URI_NORMALIZE_HOOK],[1],[Assuming htp_config_register_request_uri_normalize function in bundled libhtp])
-            AC_DEFINE_UNQUOTED([HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW],[1],[Assuming htp_tx_get_response_headers_raw function in bundled libhtp])
-            AC_DEFINE_UNQUOTED([HAVE_HTP_DECODE_QUERY_INPLACE],[1],[Assuming htp_decode_query_inplace function in bundled libhtp])
-            # enable when libhtp has been updated
-            AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT],[1],[Assuming htp_config_set_decompression_layer_limit function in bundled libhtp])
-            AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT],[1],[Assuming htp_config_set_lzma_memlimit function in bundled libhtp])
-            AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_LZMA_LAYERS],[1],[Assuming htp_config_set_lzma_layers function in bundled libhtp])
-            AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT],[1],[Assuming htp_config_set_compression_bomb_limit function in bundled libhtp])
-            AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT],[1],[Assuming htp_config_set_compression_time_limit function in bundled libhtp])
-        else
-            echo
-            echo "  ERROR: Libhtp is not bundled. Get libhtp by doing:"
-            echo "     git clone https://github.com/OISF/libhtp"
-            echo "  Then re-run Suricata's autogen.sh and configure script."
-            echo "  Or, if libhtp is installed in a different location,"
-            echo "  pass --enable-non-bundled-htp to Suricata's configure script."
-            echo "  Add --with-libhtp-includes=<dir> and --with-libhtp-libraries=<dir> if"
-            echo "  libhtp is not installed in the include and library paths."
-            echo
-            exit 1
-        fi
-    fi
-
-
   # Check for libcap-ng
     case $host in
     *-*-linux*)
@@ -2215,7 +2120,8 @@ fi
     fi
     RUST_SURICATA_LIB="${RUST_SURICATA_LIBDIR}/${RUST_SURICATA_LIBNAME}"
 
-    CFLAGS="${CFLAGS} -I\${srcdir}/../rust/gen -I\${srcdir}/../rust/dist"
+    #TODO: once LibHTP is published to crates.io point ${srcdir}../rust/ for htp/htp.h
+    CFLAGS="${CFLAGS} -I\${srcdir}/../rust/gen -I\${srcdir}/../rust/dist -I\${srcdir}/../../libhtp-rs"
     AC_SUBST(RUST_SURICATA_LIB)
     AC_SUBST(RUST_LDADD)
     if test "x$CARGO_HOME" = "x"; then
@@ -2275,6 +2181,13 @@ fi
        fi
     fi
 
+    have_htp_headers ="no"
+    AC_MSG_CHECKING(for $srcdir/rust/htp/htp.h)
+    if test -f "$srcdir/rust/htp/htp.h"; then
+       AC_MSG_RESULT(yes)
+       have_htp_headers="yes"
+    fi
+
     AC_PATH_PROG(CBINDGEN, cbindgen, "no")
     if test "x$CBINDGEN" != "xno"; then
       cbindgen_version=$(cbindgen --version 2>&1 | cut -d' ' -f2-)
@@ -2296,7 +2209,7 @@ fi
     AC_SUBST([CBINDGEN], [$CBINDGEN])
 
     # Require cbindgen if generated headers are not bundled.
-    if test "x$have_rust_headers" != "xyes"; then
+    if test "x$have_rust_headers" != "xyes" || test "x$have_htp_headers" != "xyes"; then
       if test "x$CBINDGEN" = "xno"; then
         echo "  Warning: cbindgen too old or not found, it is required to "
         echo "      generate header files."
@@ -2306,6 +2219,7 @@ fi
     fi
 
     AM_CONDITIONAL([HAVE_RUST_HEADERS], [test "x$have_rust_headers" = "xyes"])
+    AM_CONDITIONAL([HAVE_HTP_HEADERS], [test "x$have_htp_headers" = "xyes"])
     AM_CONDITIONAL([HAVE_CBINDGEN], [test "x$CBINDGEN" != "xno"])
 
     AC_ARG_ENABLE(rust_strict,
@@ -2477,7 +2391,6 @@ AC_SUBST(PACKAGE_VERSION)
 AC_SUBST(RUST_FEATURES)
 AC_SUBST(RUST_SURICATA_LIBDIR)
 AC_SUBST(RUST_SURICATA_LIBNAME)
-AC_SUBST(enable_non_bundled_htp)
 
 AM_CONDITIONAL([BUILD_SHARED_LIBRARY], [test "x$enable_shared" = "xyes"] && [test "x$can_build_shared_library" = "xyes"])
 
@@ -2516,7 +2429,6 @@ SURICATA_BUILD_CONF="Suricata Configuration:
   LUA support:                             ${enable_lua}
   libluajit:                               ${enable_luajit}
   GeoIP2 support:                          ${enable_geoip}
-  Non-bundled htp:                         ${enable_non_bundled_htp}
   Hyperscan support:                       ${enable_hyperscan}
   Libnet support:                          ${enable_libnet}
   liblz4 support:                          ${enable_liblz4}

diff --git a/rust/Cargo.toml.in b/rust/Cargo.toml.in
@@ -39,6 +39,7 @@ hkdf = "~0.12.3"
 aes = "~0.6.0"
 aes-gcm = "~0.8.0"
 
+htp = { path = "./../../libhtp-rs/", version = "2.0.0", features = ["cbindgen"] }
 sawp-modbus = "~0.11.0"
 sawp = "~0.11.0"
 der-parser = "~4.0.2"

diff --git a/rust/src/lib.rs b/rust/src/lib.rs
@@ -136,3 +136,6 @@ pub mod quic;
 pub mod plugin;
 pub mod util;
 pub mod ffi;
+
+//Re-export htp symbols
+pub use htp::c_api::*;
diff --git a/scripts/bundle.sh b/scripts/bundle.sh
@@ -28,13 +28,6 @@ while IFS= read -r requirement; do
             cp -a suricata-update.tmp/* suricata-update/
             rm -rf suricata-update.tmp
             ;;
-        libhtp)
-            repo=${LIBHTP_REPO:-$2}
-            branch=${LIBHTP_BRANCH:-$3}
-            echo "===> Bundling ${repo} -b ${branch}"
-            rm -rf libhtp
-            git clone "${repo}" -b "${branch}" libhtp
-            ;;
         \#*)
             # Ignore comment.
             ;;

diff --git a/src/Makefile.am b/src/Makefile.am
@@ -1276,7 +1276,7 @@ suricata_SOURCES = main.c
 
 # the library search path.
 suricata_LDFLAGS = $(all_libraries) ${SECLDFLAGS}
-suricata_LDADD = libsuricata_c.a $(RUST_SURICATA_LIB) $(HTP_LDADD) $(RUST_LDADD)
+suricata_LDADD = libsuricata_c.a $(RUST_SURICATA_LIB) $(RUST_LDADD)
 suricata_DEPENDENCIES = libsuricata_c.a $(RUST_SURICATA_LIB)
 
 if BUILD_SHARED_LIBRARY
@@ -1312,7 +1312,7 @@ uninstall-local:
 if BUILD_FUZZTARGETS
 
 LDFLAGS_FUZZ = $(all_libraries) $(SECLDFLAGS)
-LDADD_FUZZ = libsuricata_c.a $(RUST_SURICATA_LIB) $(HTP_LDADD) $(RUST_LDADD)
+LDADD_FUZZ = libsuricata_c.a $(RUST_SURICATA_LIB) $(RUST_LDADD)
 
 fuzz_applayerprotodetectgetproto_SOURCES = tests/fuzz/fuzz_applayerprotodetectgetproto.c
 fuzz_applayerprotodetectgetproto_LDFLAGS = $(LDFLAGS_FUZZ)

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
@@ -2383,21 +2383,12 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
     htp_config_set_plusspace_decode(cfg_prec->cfg, 0);
     // enables request decompression
     htp_config_set_request_decompression(cfg_prec->cfg, 1);
-#ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
-    // disable by default
     htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
-#endif
-#ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
     htp_config_set_lzma_memlimit(cfg_prec->cfg,
             HTP_CONFIG_DEFAULT_LZMA_MEMLIMIT);
-#endif
-#ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
     htp_config_set_compression_bomb_limit(cfg_prec->cfg,
                                           HTP_CONFIG_DEFAULT_COMPRESSION_BOMB_LIMIT);
-#endif
-#ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
     htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
-#endif
     /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
      * only the hard limit. So we set both here to the (current) htp defaults.
      * The reason we do this is that if the user sets the hard limit in the
@@ -2599,12 +2590,7 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
                            "from conf file - %s.  Killing engine", p->val);
                 exit(EXIT_FAILURE);
             }
-#ifdef HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT
             htp_config_set_decompression_layer_limit(cfg_prec->cfg, value);
-#else
-            SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
-                    "to %u, libhtp version too old", value);
-#endif
         } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
             htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
                                                      ConfValIsTrue(p->val));
@@ -2674,7 +2660,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
             /* set default soft-limit with our new hard limit */
             htp_config_set_field_limit(cfg_prec->cfg,
                     (size_t)limit);
-#ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
         } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
             uint32_t limit = 0;
             if (ParseSizeStringU32(p->val, &limit) < 0) {
@@ -2688,8 +2673,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
             /* set default soft-limit with our new hard limit */
             SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
             htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
-#endif
-#ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
         } else if (strcasecmp("lzma-enabled", p->name) == 0) {
             if (ConfValIsTrue(p->val)) {
                 htp_config_set_lzma_layers(cfg_prec->cfg, 1);
@@ -2704,8 +2687,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
                 SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
                 htp_config_set_lzma_layers(cfg_prec->cfg, limit);
             }
-#endif
-#ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
         } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
             uint32_t limit = 0;
             if (ParseSizeStringU32(p->val, &limit) < 0) {
@@ -2719,8 +2700,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
             /* set default soft-limit with our new hard limit */
             SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
             htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
-#endif
-#ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
         } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
             uint32_t limit = 0;
             // between 1 usec and 1 second
@@ -2732,7 +2711,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
             }
             SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
             htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
-#endif
         } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
             if (!g_disable_randomness) {
                 cfg_prec->randomize = ConfValIsTrue(p->val);

diff --git a/src/suricata.c b/src/suricata.c
@@ -761,9 +761,7 @@ static void PrintBuildInfo(void)
 #ifdef HAVE_LIBNET11
     strlcat(features, "LIBNET1.1 ", sizeof(features));
 #endif
-#ifdef HAVE_HTP_URI_NORMALIZE_HOOK
     strlcat(features, "HAVE_HTP_URI_NORMALIZE_HOOK ", sizeof(features));
-#endif
 #ifdef PCRE2_HAVE_JIT
     strlcat(features, "PCRE_JIT ", sizeof(features));
 #endif

diff --git a/src/tests/detect-http-server-body.c b/src/tests/detect-http-server-body.c
@@ -2621,9 +2621,7 @@ static int DetectEngineHttpServerBodyTest20(void)
     /* do detect */
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p2);
 
-#ifdef HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT
     FAIL_IF(!(PacketAlertCheck(p2, 1)));
-#endif
 
     result = 1;
 
@@ -2750,9 +2748,7 @@ static int DetectEngineHttpServerBodyTest21(void)
     /* do detect */
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p2);
 
-#ifdef HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT
     FAIL_IF(!(PacketAlertCheck(p2, 1)));
-#endif
 
     result = 1;
 
@@ -2881,9 +2877,7 @@ static int DetectEngineHttpServerBodyTest22(void)
     /* do detect */
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p2);
 
-#ifdef HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT
     FAIL_IF(!(PacketAlertCheck(p2, 1)));
-#endif
 
     result = 1;
 

diff --git a/src/util-error.c b/src/util-error.c
@@ -248,7 +248,6 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_RAWBYTES_BUFFER);
         CASE_CODE (SC_ERR_SOCKET);
         CASE_CODE (SC_ERR_PCAP_TRANSLATE);
-        CASE_CODE (SC_WARN_OUTDATED_LIBHTP);
         CASE_CODE (SC_WARN_DEPRECATED);
         CASE_CODE (SC_WARN_PROFILE);
         CASE_CODE (SC_ERR_FLOW_INIT);

diff --git a/src/util-error.h b/src/util-error.h
@@ -231,7 +231,6 @@ typedef enum {
     SC_ERR_RAWBYTES_BUFFER,
     SC_ERR_SOCKET,
     SC_ERR_PCAP_TRANSLATE, /* failed to translate ip to dev */
-    SC_WARN_OUTDATED_LIBHTP,
     SC_WARN_DEPRECATED,
     SC_WARN_PROFILE,
     SC_ERR_FLOW_INIT,