From b2417d49d67bbbdd6c9ee58f009c58098a527962 Mon Sep 17 00:00:00 2001 From: ankitoshniwal Date: Wed, 3 Jul 2013 12:17:05 -0700 Subject: [PATCH 01/15] Update nimbus.clj Passing Nimbus Conf to TopologyValidator. --- storm-core/src/clj/backtype/storm/daemon/nimbus.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj index e58aeedd0..6fbac631a 100644 --- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj +++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj @@ -907,7 +907,8 @@ (.validate ^backtype.storm.nimbus.ITopologyValidator (:validator nimbus) storm-name (from-json serializedConf) - topology) + topology + conf) (swap! (:submitted-count nimbus) inc) (let [storm-id (str storm-name "-" @(:submitted-count nimbus) "-" (current-time-secs)) storm-conf (normalize-conf From ceb002f155e599c73a86021c5dea0aec43a8c204 Mon Sep 17 00:00:00 2001 From: ankitoshniwal Date: Wed, 3 Jul 2013 12:19:03 -0700 Subject: [PATCH 02/15] Update DefaultTopologyValidator.java --- .../src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java b/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java index 6ee78fd94..94c5a0f1c 100644 --- a/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java +++ b/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java @@ -6,6 +6,6 @@ public class DefaultTopologyValidator implements ITopologyValidator { @Override - public void validate(String topologyName, Map topologyConf, StormTopology topology) throws InvalidTopologyException { + public void validate(String topologyName, Map topologyConf, StormTopology topology, Map NimbusConf) throws InvalidTopologyException { } } From aac85e221722d61ba9a2aa2395ca6a5504028a95 Mon Sep 17 00:00:00 2001 From: ankitoshniwal Date: Wed, 3 Jul 2013 12:19:51 -0700 Subject: [PATCH 03/15] Update ITopologyValidator.java --- .../src/jvm/backtype/storm/nimbus/ITopologyValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java b/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java index 9cdd1f9f0..5a19bd3a1 100644 --- a/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java +++ b/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java @@ -5,6 +5,6 @@ import java.util.Map; public interface ITopologyValidator { - void validate(String topologyName, Map topologyConf, StormTopology topology) + void validate(String topologyName, Map topologyConf, StormTopology topology, Map NimbusConf) throws InvalidTopologyException; } From ef0221979786e12c7210b2eb55f3c32c259454e8 Mon Sep 17 00:00:00 2001 From: ankitoshniwal Date: Wed, 17 Jul 2013 14:34:44 -0700 Subject: [PATCH 04/15] Update ITopologyValidator.java --- storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java b/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java index 5a19bd3a1..d9e05b6c0 100644 --- a/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java +++ b/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java @@ -5,6 +5,7 @@ import java.util.Map; public interface ITopologyValidator { + void prepare(String topologyName, Map topologyConf, StormTopology topology, Map NimbusConf); void validate(String topologyName, Map topologyConf, StormTopology topology, Map NimbusConf) throws InvalidTopologyException; } From 47d5c8d710bd08ce6f9dd4889c73f3137a7ce223 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 1 Aug 2013 02:32:35 -0700 Subject: [PATCH 05/15] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac6baf150..08c37a306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ * FixedBatchSpout will now replay batches appropriately on batch failure (thanks ptgoetz) * Can set JAR_JVM_OPTS env variable to add jvm options when calling 'storm jar' (thanks srmelody) * Throw error if batch id for transaction is behind the batch id in the opaque value (thanks mrflip) + * Added LoggingMetricsConsumer to log all metrics to a file, by default not enabled (thanks mrflip) * Bug fix: Supervisor provides full path to workers to logging config rather than relative path (thanks revans2) * Bug fix: Call ReducerAggregator#init properly when used within persistentAggregate (thanks lorcan) * Bug fix: Set component-specific configs correctly for Trident spouts From 1a27f8b81f7d9a6e1b18eec2ee45e93cf546a30d Mon Sep 17 00:00:00 2001 From: Jake Donham Date: Tue, 6 Aug 2013 10:59:14 -0700 Subject: [PATCH 06/15] sort topology summary by name --- storm-core/src/clj/backtype/storm/ui/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj index eae2072ed..d8a9d9897 100644 --- a/storm-core/src/clj/backtype/storm/ui/core.clj +++ b/storm-core/src/clj/backtype/storm/ui/core.clj @@ -103,8 +103,8 @@ (.get_num_executors t) (.get_num_tasks t) ]) - :time-cols [2] - :sort-list "[[2,1]]" + :time-cols [3] + :sort-list "[[0,0]]" )) (defn supervisor-summary-table [summs] From 1137c9074d223293caa81455210e097c4a02f55f Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 6 Aug 2013 14:56:42 -0700 Subject: [PATCH 07/15] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08c37a306..83fd45a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ * FixedBatchSpout will now replay batches appropriately on batch failure (thanks ptgoetz) * Can set JAR_JVM_OPTS env variable to add jvm options when calling 'storm jar' (thanks srmelody) * Throw error if batch id for transaction is behind the batch id in the opaque value (thanks mrflip) + * Sort topologies by name in UI (thanks jaked) * Added LoggingMetricsConsumer to log all metrics to a file, by default not enabled (thanks mrflip) * Bug fix: Supervisor provides full path to workers to logging config rather than relative path (thanks revans2) * Bug fix: Call ReducerAggregator#init properly when used within persistentAggregate (thanks lorcan) From 8b69df9afa99ea558866a6164aa12fd5bcfe4e35 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 6 Aug 2013 14:57:18 -0700 Subject: [PATCH 08/15] Update contributors --- README.markdown | 1 + 1 file changed, 1 insertion(+) diff --git a/README.markdown b/README.markdown index 0ce011c80..9c98b0f7c 100644 --- a/README.markdown +++ b/README.markdown @@ -74,6 +74,7 @@ You must not remove this notice, or any other, from this software. * Tudor Scurtu ([@tscurtu](https://github.com/tscurtu)) * Homer Strong ([@strongh](https://github.com/strongh)) * Sean Melody ([@srmelody](https://github.com/srmelody)) +* Jake Donham ([@jaked](https://github.com/jaked)) ## Acknowledgements From 08f426070eab980672b2189517e2599f33ea38b3 Mon Sep 17 00:00:00 2001 From: ankitoshniwal Date: Fri, 9 Aug 2013 14:01:30 -0700 Subject: [PATCH 09/15] Update nimbus.clj Adding ITopologyValidator prepare call during nimbus startup. --- storm-core/src/clj/backtype/storm/daemon/nimbus.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj index 6fbac631a..33bb13e11 100644 --- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj +++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj @@ -877,6 +877,7 @@ (.prepare inimbus conf (master-inimbus-dir conf)) (log-message "Starting Nimbus with conf " conf) (let [nimbus (nimbus-data conf inimbus)] + (.prepare ^backtype.storm.nimbus.ITopologyValidator (:validator nimbus) conf) (cleanup-corrupt-topologies! nimbus) (doseq [storm-id (.active-storms (:storm-cluster-state nimbus))] (transition! nimbus storm-id :startup)) From 758616c624fba00c1133ae197989d3f785495039 Mon Sep 17 00:00:00 2001 From: ankitoshniwal Date: Fri, 9 Aug 2013 14:03:13 -0700 Subject: [PATCH 10/15] Update ITopologyValidator.java Updating prepare method to accept just StormConf. --- .../src/jvm/backtype/storm/nimbus/ITopologyValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java b/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java index d9e05b6c0..52822a611 100644 --- a/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java +++ b/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java @@ -5,7 +5,7 @@ import java.util.Map; public interface ITopologyValidator { - void prepare(String topologyName, Map topologyConf, StormTopology topology, Map NimbusConf); + void prepare(Map StormConf); void validate(String topologyName, Map topologyConf, StormTopology topology, Map NimbusConf) throws InvalidTopologyException; } From 0997784c0eeea67e4e0fda1b365dce571e121e69 Mon Sep 17 00:00:00 2001 From: ankitoshniwal Date: Fri, 9 Aug 2013 14:04:29 -0700 Subject: [PATCH 11/15] Update DefaultTopologyValidator.java Adding prepare --- .../jvm/backtype/storm/nimbus/DefaultTopologyValidator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java b/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java index 94c5a0f1c..fb1716384 100644 --- a/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java +++ b/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java @@ -5,6 +5,9 @@ import java.util.Map; public class DefaultTopologyValidator implements ITopologyValidator { + @Override + public void prepare(Map StormConf){ + } @Override public void validate(String topologyName, Map topologyConf, StormTopology topology, Map NimbusConf) throws InvalidTopologyException { } From 02b61b69855f9b0475d7d3473ed29730b1c26355 Mon Sep 17 00:00:00 2001 From: ankitoshniwal Date: Fri, 9 Aug 2013 14:05:33 -0700 Subject: [PATCH 12/15] Update ITopologyValidator.java --- .../src/jvm/backtype/storm/nimbus/ITopologyValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java b/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java index 52822a611..c71413128 100644 --- a/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java +++ b/storm-core/src/jvm/backtype/storm/nimbus/ITopologyValidator.java @@ -6,6 +6,6 @@ public interface ITopologyValidator { void prepare(Map StormConf); - void validate(String topologyName, Map topologyConf, StormTopology topology, Map NimbusConf) + void validate(String topologyName, Map topologyConf, StormTopology topology) throws InvalidTopologyException; } From c3b513f8fe25164f1fac13c3bca59d04ee1b88f4 Mon Sep 17 00:00:00 2001 From: ankitoshniwal Date: Fri, 9 Aug 2013 14:20:40 -0700 Subject: [PATCH 13/15] Update nimbus.clj --- storm-core/src/clj/backtype/storm/daemon/nimbus.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj index 33bb13e11..e126a26c7 100644 --- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj +++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj @@ -908,8 +908,7 @@ (.validate ^backtype.storm.nimbus.ITopologyValidator (:validator nimbus) storm-name (from-json serializedConf) - topology - conf) + topology) (swap! (:submitted-count nimbus) inc) (let [storm-id (str storm-name "-" @(:submitted-count nimbus) "-" (current-time-secs)) storm-conf (normalize-conf From 81b5aa63c0bc17fedcf91813dd8628615d247012 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 9 Aug 2013 15:04:56 -0700 Subject: [PATCH 14/15] update changelog/contributors --- CHANGELOG.md | 1 + README.markdown | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83fd45a1b..f0d61eaa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ * Throw error if batch id for transaction is behind the batch id in the opaque value (thanks mrflip) * Sort topologies by name in UI (thanks jaked) * Added LoggingMetricsConsumer to log all metrics to a file, by default not enabled (thanks mrflip) + * Add prepare(Map conf) method to TopologyValidator (thanks ankitoshniwal) * Bug fix: Supervisor provides full path to workers to logging config rather than relative path (thanks revans2) * Bug fix: Call ReducerAggregator#init properly when used within persistentAggregate (thanks lorcan) * Bug fix: Set component-specific configs correctly for Trident spouts diff --git a/README.markdown b/README.markdown index 9c98b0f7c..5da35b09e 100644 --- a/README.markdown +++ b/README.markdown @@ -75,6 +75,7 @@ You must not remove this notice, or any other, from this software. * Homer Strong ([@strongh](https://github.com/strongh)) * Sean Melody ([@srmelody](https://github.com/srmelody)) * Jake Donham ([@jaked](https://github.com/jaked)) +* Ankit Toshniwal ([@ankitoshniwal](https://github.com/ankitoshniwal)) ## Acknowledgements From 80244bf796ac6fedcbdc74c9e2ca6b7bd58a461d Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 15 Aug 2013 22:31:58 -0700 Subject: [PATCH 15/15] fix defaulttopologyvalidator --- .../src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java b/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java index fb1716384..e92e7a149 100644 --- a/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java +++ b/storm-core/src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java @@ -9,6 +9,6 @@ public class DefaultTopologyValidator implements ITopologyValidator { public void prepare(Map StormConf){ } @Override - public void validate(String topologyName, Map topologyConf, StormTopology topology, Map NimbusConf) throws InvalidTopologyException { + public void validate(String topologyName, Map topologyConf, StormTopology topology) throws InvalidTopologyException { } }