From e7075f5a3165d7b488c15e902392990e0f5c5803 Mon Sep 17 00:00:00 2001 From: cccs-jferrant <82591883+cccs-jferrant@users.noreply.github.com> Date: Mon, 24 Oct 2022 11:01:19 -0400 Subject: [PATCH] Link to LibHTP as crate (#11) Co-authored-by: jmferra@CORP.CSE-CST.GC.CA --- configure.ac | 110 +++------------------------- rust/Cargo.toml.in | 1 + rust/src/lib.rs | 3 + scripts/bundle.sh | 7 -- src/Makefile.am | 4 +- src/app-layer-htp.c | 22 ------ src/suricata.c | 2 - src/tests/detect-http-server-body.c | 6 -- src/util-error.c | 1 - src/util-error.h | 1 - 10 files changed, 17 insertions(+), 140 deletions(-) diff --git a/configure.ac b/configure.ac index 751a9f6c27d0..c3a4a5ddca22 100644 --- a/configure.ac +++ b/configure.ac @@ -1447,101 +1447,6 @@ [test "x$install_suricata_update" = "xyes"]) AC_SUBST([install_suricata_update_reason]) - # libhtp - AC_ARG_ENABLE(non-bundled-htp, - AS_HELP_STRING([--enable-non-bundled-htp], [Enable the use of an already installed version of htp]),[enable_non_bundled_htp=$enableval],[enable_non_bundled_htp=no]) - AS_IF([test "x$enable_non_bundled_htp" = "xyes"], [ - PKG_CHECK_MODULES([libhtp], htp,, [with_pkgconfig_htp=no]) - if test "$with_pkgconfig_htp" != "no"; then - CPPFLAGS="${CPPFLAGS} ${libhtp_CFLAGS}" - LIBS="${LIBS} ${libhtp_LIBS}" - fi - - AC_ARG_WITH(libhtp_includes, - [ --with-libhtp-includes=DIR libhtp include directory], - [with_libhtp_includes="$withval"],[with_libhtp_includes=no]) - AC_ARG_WITH(libhtp_libraries, - [ --with-libhtp-libraries=DIR libhtp library directory], - [with_libhtp_libraries="$withval"],[with_libhtp_libraries="no"]) - - if test "$with_libhtp_includes" != "no"; then - CPPFLAGS="-I${with_libhtp_includes} ${CPPFLAGS}" - fi - - if test "$with_libhtp_libraries" != "no"; then - LDFLAGS="${LDFLAGS} -L${with_libhtp_libraries}" - fi - - AC_CHECK_HEADER(htp/htp.h,,[AC_MSG_ERROR(htp/htp.h not found ...)]) - - LIBHTP="" - AC_CHECK_LIB(htp, htp_conn_create,, LIBHTP="no") - if test "$LIBHTP" = "no"; then - echo - echo " ERROR! libhtp library not found" - echo - exit 1 - fi - PKG_CHECK_MODULES(LIBHTPMINVERSION, [htp >= 0.5.40],[libhtp_minver_found="yes"],[libhtp_minver_found="no"]) - if test "$libhtp_minver_found" = "no"; then - PKG_CHECK_MODULES(LIBHTPDEVVERSION, [htp = 0.5.X],[libhtp_devver_found="yes"],[libhtp_devver_found="no"]) - if test "$libhtp_devver_found" = "no"; then - echo - echo " ERROR! libhtp was found but it is neither >= 0.5.40, nor the dev 0.5.X" - echo - exit 1 - fi - fi - - AC_CHECK_LIB([htp], [htp_config_register_request_uri_normalize],AC_DEFINE_UNQUOTED([HAVE_HTP_URI_NORMALIZE_HOOK],[1],[Found htp_config_register_request_uri_normalize function in libhtp]) ,,[-lhtp]) - # check for htp_tx_get_response_headers_raw - AC_CHECK_LIB([htp], [htp_tx_get_response_headers_raw],AC_DEFINE_UNQUOTED([HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW],[1],[Found htp_tx_get_response_headers_raw in libhtp]) ,,[-lhtp]) - AC_CHECK_LIB([htp], [htp_decode_query_inplace],AC_DEFINE_UNQUOTED([HAVE_HTP_DECODE_QUERY_INPLACE],[1],[Found htp_decode_query_inplace function in libhtp]) ,,[-lhtp]) - AC_CHECK_LIB([htp], [htp_config_set_decompression_layer_limit],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT],[1],[Found htp_config_set_decompression_layer_limit function in libhtp]) ,,[-lhtp]) - AC_EGREP_HEADER(htp_config_set_path_decode_u_encoding, htp/htp.h, AC_DEFINE_UNQUOTED([HAVE_HTP_SET_PATH_DECODE_U_ENCODING],[1],[Found usable htp_config_set_path_decode_u_encoding function in libhtp]) ) - AC_CHECK_LIB([htp], [htp_config_set_lzma_memlimit],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT],[1],[Found htp_config_set_lzma_memlimit function in libhtp]) ,,[-lhtp]) - AC_CHECK_LIB([htp], [htp_config_set_lzma_layers],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_LZMA_LAYERS],[1],[Found htp_config_set_lzma_layers function in libhtp]) ,,[-lhtp]) - AC_CHECK_LIB([htp], [htp_config_set_compression_bomb_limit],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT],[1],[Found htp_config_set_compression_bomb_limit function in libhtp]) ,,[-lhtp]) - AC_CHECK_LIB([htp], [htp_config_set_compression_time_limit],AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT],[1],[Found htp_config_set_compression_time_limit function in libhtp]) ,,[-lhtp]) - ]) - - if test "x$enable_non_bundled_htp" = "xno"; then - # test if we have a bundled htp - if test -d "$srcdir/libhtp"; then - AC_CONFIG_SUBDIRS([libhtp]) - HTP_DIR="libhtp" - AC_SUBST(HTP_DIR) - HTP_LDADD="../libhtp/htp/libhtp.la" - AC_SUBST(HTP_LDADD) - # make sure libhtp is added to the includes - CPPFLAGS="-I\${srcdir}/../libhtp/ ${CPPFLAGS}" - - AC_CHECK_HEADER(iconv.h,,[AC_MSG_ERROR(iconv.h not found ...)]) - AC_CHECK_LIB(iconv, libiconv_close) - AC_DEFINE_UNQUOTED([HAVE_HTP_URI_NORMALIZE_HOOK],[1],[Assuming htp_config_register_request_uri_normalize function in bundled libhtp]) - AC_DEFINE_UNQUOTED([HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW],[1],[Assuming htp_tx_get_response_headers_raw function in bundled libhtp]) - AC_DEFINE_UNQUOTED([HAVE_HTP_DECODE_QUERY_INPLACE],[1],[Assuming htp_decode_query_inplace function in bundled libhtp]) - # enable when libhtp has been updated - AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT],[1],[Assuming htp_config_set_decompression_layer_limit function in bundled libhtp]) - AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT],[1],[Assuming htp_config_set_lzma_memlimit function in bundled libhtp]) - AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_LZMA_LAYERS],[1],[Assuming htp_config_set_lzma_layers function in bundled libhtp]) - AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT],[1],[Assuming htp_config_set_compression_bomb_limit function in bundled libhtp]) - AC_DEFINE_UNQUOTED([HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT],[1],[Assuming htp_config_set_compression_time_limit function in bundled libhtp]) - else - echo - echo " ERROR: Libhtp is not bundled. Get libhtp by doing:" - echo " git clone https://github.com/OISF/libhtp" - echo " Then re-run Suricata's autogen.sh and configure script." - echo " Or, if libhtp is installed in a different location," - echo " pass --enable-non-bundled-htp to Suricata's configure script." - echo " Add --with-libhtp-includes= and --with-libhtp-libraries= if" - echo " libhtp is not installed in the include and library paths." - echo - exit 1 - fi - fi - - # Check for libcap-ng case $host in *-*-linux*) @@ -2215,7 +2120,8 @@ fi fi RUST_SURICATA_LIB="${RUST_SURICATA_LIBDIR}/${RUST_SURICATA_LIBNAME}" - CFLAGS="${CFLAGS} -I\${srcdir}/../rust/gen -I\${srcdir}/../rust/dist" + #TODO: once LibHTP is published to crates.io point ${srcdir}../rust/ for htp/htp.h + CFLAGS="${CFLAGS} -I\${srcdir}/../rust/gen -I\${srcdir}/../rust/dist -I\${srcdir}/../../libhtp-rs" AC_SUBST(RUST_SURICATA_LIB) AC_SUBST(RUST_LDADD) if test "x$CARGO_HOME" = "x"; then @@ -2275,6 +2181,13 @@ fi fi fi + have_htp_headers ="no" + AC_MSG_CHECKING(for $srcdir/rust/htp/htp.h) + if test -f "$srcdir/rust/htp/htp.h"; then + AC_MSG_RESULT(yes) + have_htp_headers="yes" + fi + AC_PATH_PROG(CBINDGEN, cbindgen, "no") if test "x$CBINDGEN" != "xno"; then cbindgen_version=$(cbindgen --version 2>&1 | cut -d' ' -f2-) @@ -2296,7 +2209,7 @@ fi AC_SUBST([CBINDGEN], [$CBINDGEN]) # Require cbindgen if generated headers are not bundled. - if test "x$have_rust_headers" != "xyes"; then + if test "x$have_rust_headers" != "xyes" || test "x$have_htp_headers" != "xyes"; then if test "x$CBINDGEN" = "xno"; then echo " Warning: cbindgen too old or not found, it is required to " echo " generate header files." @@ -2306,6 +2219,7 @@ fi fi AM_CONDITIONAL([HAVE_RUST_HEADERS], [test "x$have_rust_headers" = "xyes"]) + AM_CONDITIONAL([HAVE_HTP_HEADERS], [test "x$have_htp_headers" = "xyes"]) AM_CONDITIONAL([HAVE_CBINDGEN], [test "x$CBINDGEN" != "xno"]) AC_ARG_ENABLE(rust_strict, @@ -2477,7 +2391,6 @@ AC_SUBST(PACKAGE_VERSION) AC_SUBST(RUST_FEATURES) AC_SUBST(RUST_SURICATA_LIBDIR) AC_SUBST(RUST_SURICATA_LIBNAME) -AC_SUBST(enable_non_bundled_htp) AM_CONDITIONAL([BUILD_SHARED_LIBRARY], [test "x$enable_shared" = "xyes"] && [test "x$can_build_shared_library" = "xyes"]) @@ -2516,7 +2429,6 @@ SURICATA_BUILD_CONF="Suricata Configuration: LUA support: ${enable_lua} libluajit: ${enable_luajit} GeoIP2 support: ${enable_geoip} - Non-bundled htp: ${enable_non_bundled_htp} Hyperscan support: ${enable_hyperscan} Libnet support: ${enable_libnet} liblz4 support: ${enable_liblz4} diff --git a/rust/Cargo.toml.in b/rust/Cargo.toml.in index 4a4990a57dd2..a5cbec196b02 100644 --- a/rust/Cargo.toml.in +++ b/rust/Cargo.toml.in @@ -39,6 +39,7 @@ hkdf = "~0.12.3" aes = "~0.6.0" aes-gcm = "~0.8.0" +htp = { path = "./../../libhtp-rs/", version = "2.0.0", features = ["cbindgen"] } sawp-modbus = "~0.11.0" sawp = "~0.11.0" der-parser = "~4.0.2" diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a8b729bf6103..b40e89e2e944 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -136,3 +136,6 @@ pub mod quic; pub mod plugin; pub mod util; pub mod ffi; + +//Re-export htp symbols +pub use htp::c_api::*; diff --git a/scripts/bundle.sh b/scripts/bundle.sh index 46c0c0500cd9..7dad141b3703 100755 --- a/scripts/bundle.sh +++ b/scripts/bundle.sh @@ -28,13 +28,6 @@ while IFS= read -r requirement; do cp -a suricata-update.tmp/* suricata-update/ rm -rf suricata-update.tmp ;; - libhtp) - repo=${LIBHTP_REPO:-$2} - branch=${LIBHTP_BRANCH:-$3} - echo "===> Bundling ${repo} -b ${branch}" - rm -rf libhtp - git clone "${repo}" -b "${branch}" libhtp - ;; \#*) # Ignore comment. ;; diff --git a/src/Makefile.am b/src/Makefile.am index e28d635dc4ca..aabe3e5b377c 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1276,7 +1276,7 @@ suricata_SOURCES = main.c # the library search path. suricata_LDFLAGS = $(all_libraries) ${SECLDFLAGS} -suricata_LDADD = libsuricata_c.a $(RUST_SURICATA_LIB) $(HTP_LDADD) $(RUST_LDADD) +suricata_LDADD = libsuricata_c.a $(RUST_SURICATA_LIB) $(RUST_LDADD) suricata_DEPENDENCIES = libsuricata_c.a $(RUST_SURICATA_LIB) if BUILD_SHARED_LIBRARY @@ -1312,7 +1312,7 @@ uninstall-local: if BUILD_FUZZTARGETS LDFLAGS_FUZZ = $(all_libraries) $(SECLDFLAGS) -LDADD_FUZZ = libsuricata_c.a $(RUST_SURICATA_LIB) $(HTP_LDADD) $(RUST_LDADD) +LDADD_FUZZ = libsuricata_c.a $(RUST_SURICATA_LIB) $(RUST_LDADD) fuzz_applayerprotodetectgetproto_SOURCES = tests/fuzz/fuzz_applayerprotodetectgetproto.c fuzz_applayerprotodetectgetproto_LDFLAGS = $(LDFLAGS_FUZZ) diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 7e49871c1d90..5119600baeb7 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -2383,21 +2383,12 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec) htp_config_set_plusspace_decode(cfg_prec->cfg, 0); // enables request decompression htp_config_set_request_decompression(cfg_prec->cfg, 1); -#ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS - // disable by default htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS); -#endif -#ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT htp_config_set_lzma_memlimit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_MEMLIMIT); -#endif -#ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT htp_config_set_compression_bomb_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_BOMB_LIMIT); -#endif -#ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT); -#endif /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set * only the hard limit. So we set both here to the (current) htp defaults. * The reason we do this is that if the user sets the hard limit in the @@ -2599,12 +2590,7 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s, "from conf file - %s. Killing engine", p->val); exit(EXIT_FAILURE); } -#ifdef HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT htp_config_set_decompression_layer_limit(cfg_prec->cfg, value); -#else - SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit " - "to %u, libhtp version too old", value); -#endif } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) { htp_config_set_backslash_convert_slashes(cfg_prec->cfg, ConfValIsTrue(p->val)); @@ -2674,7 +2660,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s, /* set default soft-limit with our new hard limit */ htp_config_set_field_limit(cfg_prec->cfg, (size_t)limit); -#ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT } else if (strcasecmp("lzma-memlimit", p->name) == 0) { uint32_t limit = 0; if (ParseSizeStringU32(p->val, &limit) < 0) { @@ -2688,8 +2673,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s, /* set default soft-limit with our new hard limit */ SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit); htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit); -#endif -#ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS } else if (strcasecmp("lzma-enabled", p->name) == 0) { if (ConfValIsTrue(p->val)) { htp_config_set_lzma_layers(cfg_prec->cfg, 1); @@ -2704,8 +2687,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s, SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit); htp_config_set_lzma_layers(cfg_prec->cfg, limit); } -#endif -#ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT } else if (strcasecmp("compression-bomb-limit", p->name) == 0) { uint32_t limit = 0; if (ParseSizeStringU32(p->val, &limit) < 0) { @@ -2719,8 +2700,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s, /* set default soft-limit with our new hard limit */ SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit); htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit); -#endif -#ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT } else if (strcasecmp("decompression-time-limit", p->name) == 0) { uint32_t limit = 0; // between 1 usec and 1 second @@ -2732,7 +2711,6 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s, } SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit); htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit); -#endif } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) { if (!g_disable_randomness) { cfg_prec->randomize = ConfValIsTrue(p->val); diff --git a/src/suricata.c b/src/suricata.c index 63dfd572da97..162ec84ffbf3 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -761,9 +761,7 @@ static void PrintBuildInfo(void) #ifdef HAVE_LIBNET11 strlcat(features, "LIBNET1.1 ", sizeof(features)); #endif -#ifdef HAVE_HTP_URI_NORMALIZE_HOOK strlcat(features, "HAVE_HTP_URI_NORMALIZE_HOOK ", sizeof(features)); -#endif #ifdef PCRE2_HAVE_JIT strlcat(features, "PCRE_JIT ", sizeof(features)); #endif diff --git a/src/tests/detect-http-server-body.c b/src/tests/detect-http-server-body.c index 6983db40e84e..c29a09d400b8 100644 --- a/src/tests/detect-http-server-body.c +++ b/src/tests/detect-http-server-body.c @@ -2621,9 +2621,7 @@ static int DetectEngineHttpServerBodyTest20(void) /* do detect */ SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); -#ifdef HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT FAIL_IF(!(PacketAlertCheck(p2, 1))); -#endif result = 1; @@ -2750,9 +2748,7 @@ static int DetectEngineHttpServerBodyTest21(void) /* do detect */ SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); -#ifdef HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT FAIL_IF(!(PacketAlertCheck(p2, 1))); -#endif result = 1; @@ -2881,9 +2877,7 @@ static int DetectEngineHttpServerBodyTest22(void) /* do detect */ SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); -#ifdef HAVE_HTP_CONFIG_SET_DECOMPRESSION_LAYER_LIMIT FAIL_IF(!(PacketAlertCheck(p2, 1))); -#endif result = 1; diff --git a/src/util-error.c b/src/util-error.c index 5a6976d0213a..0d655f2c3586 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -248,7 +248,6 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_RAWBYTES_BUFFER); CASE_CODE (SC_ERR_SOCKET); CASE_CODE (SC_ERR_PCAP_TRANSLATE); - CASE_CODE (SC_WARN_OUTDATED_LIBHTP); CASE_CODE (SC_WARN_DEPRECATED); CASE_CODE (SC_WARN_PROFILE); CASE_CODE (SC_ERR_FLOW_INIT); diff --git a/src/util-error.h b/src/util-error.h index 42508f4b21e1..cd76d65a7d06 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -231,7 +231,6 @@ typedef enum { SC_ERR_RAWBYTES_BUFFER, SC_ERR_SOCKET, SC_ERR_PCAP_TRANSLATE, /* failed to translate ip to dev */ - SC_WARN_OUTDATED_LIBHTP, SC_WARN_DEPRECATED, SC_WARN_PROFILE, SC_ERR_FLOW_INIT,