Merge pull request #851 from tempesta-tech/ik-51

Live reconfiguration

etc/tempesta_fw.conf

@@ -185,6 +185,23 @@
 #   server_queue_size 1000;
 #
 
+#
+# TAG: grace_shutdown_time
+#
+# Defines the timeout in seconds for graceful server removing during
+# reconfiguration.
+#
+# Syntax:
+#   grace_shutdown_time 5;
+#
+# If a server is deleted from the configuration, but is still alive, then
+# let all current sessions and requests to finish, but do not schedule new
+# sessions/requests to the server.
+#
+# Default:
+#   grace_shutdown_time 0;
+#
+
 # TAG: srv_group
 #
 # Groups multiple backend servers into a single unit of load balancing.

pkg/debian/tempesta-fw.service

@@ -14,7 +14,7 @@ Environment=TFW_CFG_PATH=/etc/tempesta_fw.conf
 RemainAfterExit=yes
 ExecStart=/lib/tempesta/scripts/tempesta.sh --start
 ExecStop=/lib/tempesta/scripts/tempesta.sh --stop
-ExecReload=/lib/tempesta/scripts/tempesta.sh --restart
+ExecReload=/lib/tempesta/scripts/tempesta.sh --reload
 
 [Install]
 WantedBy=multi-user.target

scripts/tempesta.sh

@@ -3,7 +3,7 @@
 # Tempesta FW service script.
 #
 # Copyright (C) 2014 NatSys Lab. ([email protected]).
-# Copyright (C) 2015-2016 Tempesta Technologies, Inc.
+# Copyright (C) 2015-2017 Tempesta Technologies, Inc.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -35,7 +35,7 @@ tfw_mod=tempesta_fw
 tfw_sched_mod=tfw_sched_$sched
 frang_mod="tfw_frang"
 declare frang_enable=
-declare -r LONG_OPTS="help,load,unload,start,stop,restart"
+declare -r LONG_OPTS="help,load,unload,start,stop,restart,reload"
 
 declare devs=$(ip addr show up | awk '/^[0-9]+/ { sub(/:/, "", $2); print $2}')
 
@@ -53,6 +53,7 @@ usage()
 	echo -e "  --start     Load modules and start."
 	echo -e "  --stop      Stop and unload modules."
 	echo -e "  --restart   Restart.\n"
+	echo -e "  --reload    Live reconfiguration.\n"
 }
 
 error()
@@ -64,6 +65,22 @@ error()
 # Tempesta requires kernel module loading, so we need root credentials.
 [ `id -u` -ne 0 ] && error "Please, run the script as root"
 
+load_one_module()
+{
+	if [ -z "$1" ]; then
+		echo "$0: Empty argument";
+		exit 255;
+	fi
+
+	MOD_PATH_NAME="$1"; shift;
+	MOD_NAME="$(basename ${MOD_PATH_NAME%%.*})";
+
+	lsmod | grep -w "${MOD_NAME}" 2>&1 > /dev/null || {
+		echo "Loading module ${MOD_NAME} $@";
+		insmod "${MOD_PATH_NAME}" "$@";
+	}
+}
+
 # The separate load_modules/unload_modules routines are used for unit testing.
 load_modules()
 {
@@ -73,24 +90,24 @@ load_modules()
 	# so debug messages are shown on serial console as well.
 	echo '8 7 1 7' > /proc/sys/kernel/printk
 
-	insmod $tls_path/$tls_mod.ko
-	[ $? -ne 0 ] && error "cannot load tempesta TLS module"
+	load_one_module "$tls_path/$tls_mod.ko" ||
+		error "cannot load tempesta TLS module"
 
-	insmod $tdb_path/$tdb_mod.ko
-	[ $? -ne 0 ] && error "cannot load tempesta database module"
+	load_one_module "$tdb_path/$tdb_mod.ko" ||
+		error "cannot load tempesta database module"
 
-	insmod $tfw_path/$tfw_mod.ko tfw_cfg_path=$tfw_cfg_path
-	[ $? -ne 0 ] && error "cannot load tempesta module"
+	load_one_module "$tfw_path/$tfw_mod.ko" "tfw_cfg_path=$tfw_cfg_path" ||
+		error "cannot load tempesta module"
 
 	for ko_file in "${sched_ko_files[@]}"; do
-		insmod $ko_file
-		[ $? -ne 0 ] && error "cannot load tempesta scheduler module"
+		load_one_module "$ko_file" ||
+			error "cannot load tempesta scheduler module"
 	done
 
 	if [ "$frang_enable" ]; then
 		echo "Load Frang"
-		insmod $class_path/$frang_mod.ko
-		[ $? -ne 0 ] && error "cannot load $frang_mod module"
+		load_one_module "$class_path/$frang_mod.ko" ||
+			error "cannot load $frang_mod module"
 	fi
 }
 
@@ -108,10 +125,8 @@ unload_modules()
 	rmmod $tls_mod
 }
 
-start()
+setup()
 {
-	echo "Starting Tempesta..."
-
 	tfw_set_net_queues "$devs"
 
 	# Tempesta builds socket buffers by itself, don't cork TCP segments.
@@ -120,16 +135,28 @@ start()
 	sysctl -w net.core.netdev_max_backlog=10000 >/dev/null
 	sysctl -w net.core.somaxconn=131072 >/dev/null
 	sysctl -w net.ipv4.tcp_max_syn_backlog=131072 >/dev/null
+}
+
+start()
+{
+	echo "Starting Tempesta..."
+
+	TFW_STATE=$(sysctl net.tempesta.state 2> /dev/null)
+	TFW_STATE=${TFW_STATE##* }
 
-	echo "...load Tempesta modules"
-	load_modules
+	[[ -z ${TFW_STATE} ]] && {
+		setup;
 
-	# Create database directory if it doesn't exist.
-	mkdir -p /opt/tempesta/db/
-	# At this time we don't have stable TDB data format, so
-	# it would be nice to clean all the tables before the start.
-	# TODO: Remove the hack when TDB is fixed.
-	rm -f /opt/tempesta/db/*.tdb
+		echo "...load Tempesta modules"
+		load_modules;
+
+		# Create database directory if it doesn't exist.
+		mkdir -p /opt/tempesta/db/;
+		# At this time we don't have stable TDB data format, so
+		# it would be nice to clean all the tables before the start.
+		# TODO: Remove the hack when TDB is fixed.
+		rm -f /opt/tempesta/db/*.tdb;
+	}
 
 	echo "...start Tempesta FW"
 	sysctl -w net.tempesta.state=start >/dev/null
@@ -153,6 +180,17 @@ stop()
 	echo "done"
 }
 
+reload()
+{
+	echo "Running live reconfiguration of Tempesta..."
+	sysctl -w net.tempesta.state=start >/dev/null
+	if [ $? -ne 0 ]; then
+		error "cannot reconfigure Tempesta FW"
+	else
+		echo "done"
+	fi
+}
+
 args=$(getopt -o "d:f" -a -l "$LONG_OPTS" -- "$@")
 eval set -- "${args}"
 while :; do
@@ -180,6 +218,10 @@ while :; do
 			start
 			exit
 			;;
+		--reload)
+			reload
+			exit
+			;;
 		# Ignore any options after action.
 		-d)
 			devs=$2

tempesta_fw/apm.c

@@ -946,7 +946,7 @@ tfw_apm_rbent_init(TfwApmRBEnt *rbent, unsigned long jtmistamp)
  * Note that due to specifics of Tempesta start up process this code
  * is executed in SoftIRQ context (so that sleeping is not allowed).
  */
-void *
+static void *
 tfw_apm_create(void)
 {
 	TfwApmData *data;
@@ -1060,23 +1060,19 @@ tfw_apm_del_srv(TfwServer *srv)
 
 #define TFW_APM_MIN_TMWSCALE	1	/* Minimum time window scale. */
 #define TFW_APM_MAX_TMWSCALE	50	/* Maximum time window scale. */
-#define TFW_APM_DEF_TMWSCALE	5	/* Default time window scale. */
 
 #define TFW_APM_MIN_TMWINDOW	60	/* Minimum time window (secs). */
 #define TFW_APM_MAX_TMWINDOW	3600	/* Maximum time window (secs). */
-#define TFW_APM_DEF_TMWINDOW	300	/* Default time window (secs). */
 
 #define TFW_APM_MIN_TMINTRVL	5	/* Minimum time interval (secs). */
 
 static int
-tfw_apm_cfg_start(void)
+tfw_apm_cfgend(void)
 {
 	unsigned int jtmwindow;
 
-	if (!tfw_apm_jtmwindow)
-		tfw_apm_jtmwindow = TFW_APM_DEF_TMWINDOW;
-	if (!tfw_apm_tmwscale)
-		tfw_apm_tmwscale = TFW_APM_DEF_TMWSCALE;
+	if (tfw_runstate_is_reconfig())
+		return 0;
 
 	if ((tfw_apm_jtmwindow < TFW_APM_MIN_TMWINDOW)
 	    || (tfw_apm_jtmwindow > TFW_APM_MAX_TMWINDOW))
@@ -1116,13 +1112,13 @@ tfw_apm_cfg_start(void)
  * and the APM timers are deleted.
  */
 static void
-tfw_apm_cfg_cleanup(TfwCfgSpec *cs)
+tfw_cfgop_cleanup_apm(TfwCfgSpec *cs)
 {
-	tfw_apm_jtmwindow = tfw_apm_jtmintrvl = tfw_apm_tmwscale = 0;
+	tfw_apm_jtmwindow = tfw_apm_tmwscale = 0;
 }
 
 static int
-tfw_handle_apm_stats(TfwCfgSpec *cs, TfwCfgEntry *ce)
+tfw_cfgop_apm_stats(TfwCfgSpec *cs, TfwCfgEntry *ce)
 {
 	int i, r;
 	const char *key, *val;
@@ -1137,6 +1133,7 @@ tfw_handle_apm_stats(TfwCfgSpec *cs, TfwCfgEntry *ce)
 			    cs->name);
 		return 0;
 	}
+
 	TFW_CFG_ENTRY_FOR_EACH_ATTR(ce, i, key, val) {
 		if (!strcasecmp(key, "window")) {
 			if ((r = tfw_cfg_parse_int(val, &tfw_apm_jtmwindow)))
@@ -1154,19 +1151,33 @@ tfw_handle_apm_stats(TfwCfgSpec *cs, TfwCfgEntry *ce)
 	return 0;
 }
 
-static TfwCfgSpec tfw_apm_cfg_specs[] = {
+static TfwCfgSpec tfw_apm_specs[] = {
 	{
-		"apm_stats", NULL,
-		tfw_handle_apm_stats,
+		.name = "apm_stats",
+		.deflt = "window=300 scale=5",
+		.handler = tfw_cfgop_apm_stats,
+		.cleanup  = tfw_cfgop_cleanup_apm,
 		.allow_none = true,
 		.allow_repeat = false,
-		.cleanup  = tfw_apm_cfg_cleanup,
 	},
-	{}
+	{ 0 }
 };
 
-TfwCfgMod tfw_apm_cfg_mod = {
-	.name  = "apm",
-	.start = tfw_apm_cfg_start,
-	.specs = tfw_apm_cfg_specs,
+TfwMod tfw_apm_mod = {
+	.name	= "apm",
+	.cfgend	= tfw_apm_cfgend,
+	.specs	= tfw_apm_specs,
 };
+
+int
+tfw_apm_init(void)
+{
+	tfw_mod_register(&tfw_apm_mod);
+	return 0;
+}
+
+void
+tfw_apm_exit(void)
+{
+	tfw_mod_unregister(&tfw_apm_mod);
+}