[GitHub] metron pull request #888: METRON-1389: Zeppelin notebook import does not wor...
Github user anandsubbu commented on a diff in the pull request: https://github.com/apache/metron/pull/888#discussion_r161955419 --- Diff: metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py --- @@ -162,9 +163,21 @@ def zeppelin_notebook_import(self, env): --- End diff -- In Ambari, the Shiro INI content appears as a template like here: https://user-images.githubusercontent.com/20395490/35026106-0a038a28-fb6f-11e7-93e3-371e5fa353d0.png";> And this can be fetched using Ambari API as follows: http://:8080/api/v1/clusters/cl1/configurations?type=zeppelin-shiro-ini&tag=version1 Which I am fetching in the `status_params.py` as ``` zeppelin_shiro_ini_content = config['configurations']['zeppelin-shiro-ini']['shiro_ini_content'] ``` I found it easy to parse this from the content. Could you please clarify how I can use `dict` to get this info? ---
[GitHub] metron issue #899: METRON-1405: Add Boyer-Moore majority vote algorithm to S...
Github user ottobackwards commented on the issue: https://github.com/apache/metron/pull/899 Just in case: Note that that test is modified from your test ( _o ) ---
[GitHub] metron pull request #899: METRON-1405: Add Boyer-Moore majority vote algorit...
Github user ottobackwards commented on a diff in the pull request: https://github.com/apache/metron/pull/899#discussion_r161919259 --- Diff: metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java --- @@ -220,4 +219,156 @@ public Object apply(List args) { } } } + + @Stellar(namespace = "BOYERMOORE" + , name = "ADD" + , description = "Adds value to a Boyer-Moore list. [Boyer-Moore](https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm)" + , params = { +"state - state holder for list of values. If null, add will initialize a new state value.", +"value(s) - single object or list of values to add to the state object." + } + , returns = "Current state of the Boyer-Moore algorithm representing the current value that" + + "holds a plurality across all values added thus far." + ) + public static class BoyerMooreAdd extends BaseStellarFunction { + +@Override +public Object apply(List args) { + if (args.size() < 2) { +throw new IllegalArgumentException( +"Must pass an initial state (may be null) and at least one value to add to the list"); + } else { +BoyerMooreState state = ConversionUtils.convert(args.get(0), BoyerMooreState.class); +if (state == null) { + state = new BoyerMooreState(); +} +Object secondArg = args.get(1); +if (secondArg instanceof List) { + state.addAll((List) secondArg); +} else { + state.add(secondArg); +} +return state; + } +} + } + + public static class BoyerMooreState { +private Long counter; +private Object m; + +public BoyerMooreState() { + counter = 0L; +} + +public BoyerMooreState(Optional> previousStates, Optional currentState) { + this(); + currentState.ifPresent(boyerMooreState -> { +m = boyerMooreState.getPlurality(); +counter = boyerMooreState.getCounter(); + }); + for (BoyerMooreState state : previousStates.orElse(new ArrayList<>())) { +Object plurality = state.getPlurality(); +Long pluralityCount = state.getCounter(); +add(plurality, pluralityCount); + } +} + +public Object add(Object item) { + if (item != null) { +if (counter == 0) { + m = item; + counter = 1L; +} else if (item.equals(m)) { + counter++; +} else { + counter--; +} + } + return m; +} + +public Object add(Object item, Long counter) { + if (item != null) { +if (this.counter == 0) { + m = item; + this.counter = counter; --- End diff -- no, i'm talking about object.equals(object2) when the type of the objects is determined by antlr etc ---
[GitHub] metron issue #899: METRON-1405: Add Boyer-Moore majority vote algorithm to S...
Github user mmiklavc commented on the issue: https://github.com/apache/metron/pull/899 I'll look into the test as it's passing locally for me in my IDE. ---
[GitHub] metron pull request #899: METRON-1405: Add Boyer-Moore majority vote algorit...
Github user mmiklavc commented on a diff in the pull request: https://github.com/apache/metron/pull/899#discussion_r161913450 --- Diff: metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java --- @@ -220,4 +219,156 @@ public Object apply(List args) { } } } + + @Stellar(namespace = "BOYERMOORE" + , name = "ADD" + , description = "Adds value to a Boyer-Moore list. [Boyer-Moore](https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm)" + , params = { +"state - state holder for list of values. If null, add will initialize a new state value.", +"value(s) - single object or list of values to add to the state object." + } + , returns = "Current state of the Boyer-Moore algorithm representing the current value that" + + "holds a plurality across all values added thus far." + ) + public static class BoyerMooreAdd extends BaseStellarFunction { + +@Override +public Object apply(List args) { + if (args.size() < 2) { +throw new IllegalArgumentException( +"Must pass an initial state (may be null) and at least one value to add to the list"); + } else { +BoyerMooreState state = ConversionUtils.convert(args.get(0), BoyerMooreState.class); +if (state == null) { + state = new BoyerMooreState(); +} +Object secondArg = args.get(1); +if (secondArg instanceof List) { + state.addAll((List) secondArg); +} else { + state.add(secondArg); +} +return state; + } +} + } + + public static class BoyerMooreState { +private Long counter; +private Object m; + +public BoyerMooreState() { + counter = 0L; +} + +public BoyerMooreState(Optional> previousStates, Optional currentState) { + this(); + currentState.ifPresent(boyerMooreState -> { +m = boyerMooreState.getPlurality(); +counter = boyerMooreState.getCounter(); + }); + for (BoyerMooreState state : previousStates.orElse(new ArrayList<>())) { +Object plurality = state.getPlurality(); +Long pluralityCount = state.getCounter(); +add(plurality, pluralityCount); + } +} + +public Object add(Object item) { + if (item != null) { +if (counter == 0) { + m = item; + counter = 1L; +} else if (item.equals(m)) { + counter++; +} else { + counter--; +} + } + return m; +} + +public Object add(Object item, Long counter) { + if (item != null) { +if (this.counter == 0) { + m = item; + this.counter = counter; --- End diff -- AFA rounding errors - the counter you're seeing there are internal state. In the case of merging multiple states together, it will effectively merge the counts to nominate a new plurality leader. If we could do this in 2 passes, you'd have a guaranteed majority, but I thought it was still useful to see what item has the most without necessarily being a majority. I'll add some use case documentation this week as I think this might be useful in the profiler. ---
[GitHub] metron pull request #899: METRON-1405: Add Boyer-Moore majority vote algorit...
Github user mmiklavc commented on a diff in the pull request: https://github.com/apache/metron/pull/899#discussion_r161912600 --- Diff: metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java --- @@ -220,4 +219,156 @@ public Object apply(List args) { } } } + + @Stellar(namespace = "BOYERMOORE" + , name = "ADD" + , description = "Adds value to a Boyer-Moore list. [Boyer-Moore](https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm)" + , params = { +"state - state holder for list of values. If null, add will initialize a new state value.", +"value(s) - single object or list of values to add to the state object." + } + , returns = "Current state of the Boyer-Moore algorithm representing the current value that" + + "holds a plurality across all values added thus far." + ) + public static class BoyerMooreAdd extends BaseStellarFunction { + +@Override +public Object apply(List args) { + if (args.size() < 2) { +throw new IllegalArgumentException( +"Must pass an initial state (may be null) and at least one value to add to the list"); + } else { +BoyerMooreState state = ConversionUtils.convert(args.get(0), BoyerMooreState.class); +if (state == null) { + state = new BoyerMooreState(); +} +Object secondArg = args.get(1); +if (secondArg instanceof List) { + state.addAll((List) secondArg); +} else { + state.add(secondArg); +} +return state; + } +} + } + + public static class BoyerMooreState { +private Long counter; +private Object m; + +public BoyerMooreState() { + counter = 0L; +} + +public BoyerMooreState(Optional> previousStates, Optional currentState) { + this(); + currentState.ifPresent(boyerMooreState -> { +m = boyerMooreState.getPlurality(); +counter = boyerMooreState.getCounter(); + }); + for (BoyerMooreState state : previousStates.orElse(new ArrayList<>())) { +Object plurality = state.getPlurality(); +Long pluralityCount = state.getCounter(); +add(plurality, pluralityCount); + } +} + +public Object add(Object item) { + if (item != null) { +if (counter == 0) { + m = item; + counter = 1L; +} else if (item.equals(m)) { + counter++; +} else { + counter--; +} + } + return m; +} + +public Object add(Object item, Long counter) { + if (item != null) { +if (this.counter == 0) { + m = item; + this.counter = counter; --- End diff -- It's true, I make no effort to massage data types here - if you want to interpret a 1 as a string "1", then they will be considered distinct. I didn't have a suitable mechanism for determining types. I thought it was reasonable for a user to be able to make that type cast as part of their Stellar expression building, but I'm certainly open to suggestions. ---
[GitHub] metron pull request #899: METRON-1405: Add Boyer-Moore majority vote algorit...
Github user mmiklavc commented on a diff in the pull request: https://github.com/apache/metron/pull/899#discussion_r161911833 --- Diff: metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java --- @@ -220,4 +219,156 @@ public Object apply(List args) { } } } + + @Stellar(namespace = "BOYERMOORE" + , name = "ADD" + , description = "Adds value to a Boyer-Moore list. [Boyer-Moore](https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm)" + , params = { +"state - state holder for list of values. If null, add will initialize a new state value.", +"value(s) - single object or list of values to add to the state object." + } + , returns = "Current state of the Boyer-Moore algorithm representing the current value that" + + "holds a plurality across all values added thus far." + ) + public static class BoyerMooreAdd extends BaseStellarFunction { + +@Override +public Object apply(List args) { + if (args.size() < 2) { +throw new IllegalArgumentException( +"Must pass an initial state (may be null) and at least one value to add to the list"); + } else { +BoyerMooreState state = ConversionUtils.convert(args.get(0), BoyerMooreState.class); +if (state == null) { + state = new BoyerMooreState(); +} +Object secondArg = args.get(1); +if (secondArg instanceof List) { + state.addAll((List) secondArg); +} else { + state.add(secondArg); +} +return state; + } +} + } + + public static class BoyerMooreState { +private Long counter; +private Object m; + +public BoyerMooreState() { + counter = 0L; +} + +public BoyerMooreState(Optional> previousStates, Optional currentState) { + this(); + currentState.ifPresent(boyerMooreState -> { +m = boyerMooreState.getPlurality(); +counter = boyerMooreState.getCounter(); + }); + for (BoyerMooreState state : previousStates.orElse(new ArrayList<>())) { +Object plurality = state.getPlurality(); +Long pluralityCount = state.getCounter(); +add(plurality, pluralityCount); + } +} + +public Object add(Object item) { + if (item != null) { +if (counter == 0) { + m = item; + counter = 1L; +} else if (item.equals(m)) { + counter++; +} else { + counter--; +} + } + return m; +} + +public Object add(Object item, Long counter) { + if (item != null) { +if (this.counter == 0) { + m = item; + this.counter = counter; +} else if (item.equals(m)) { + this.counter += counter; +} else if (counter > this.counter) { + m = item; + this.counter = counter - this.counter; +} else { + this.counter = 0L; +} + } + return m; +} + +public Object getPlurality() { + return m; +} + +public Object addAll(List items) { + for (Object item : items) { +add(item); + } + return m; +} + +public Long getCounter() { + return counter; +} + } + + @Stellar(namespace = "BOYERMOORE" + , name = "PLURALITY" + , description = "Calculates the item with current plurality in a Boyer-Moore list. [Boyer-Moore](https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm)" + , params = { + "state - state holder for list of values. If null, add will initialize a new state value." + } --- End diff -- Good catch ---
[GitHub] metron issue #899: METRON-1405: Add Boyer-Moore majority vote algorithm to S...
Github user ottobackwards commented on the issue: https://github.com/apache/metron/pull/899 This tests fails, I think something is wrong. ```java @Test public void boyerMoore_calculates_plurality_from_list_of_mixed_objects_o() throws Exception { List items = Arrays.asList(1,1,1,1, "orange", "orange", "jello", "jello", "jello"); BoyerMooreState state = (BoyerMooreState) new BoyerMooreAdd().apply(Arrays.asList(null, items)); Assert.assertThat(new BoyerMoorePlurality().apply(ImmutableList.of(state)), CoreMatchers.equalTo(1)); } ``` ---
[GitHub] metron issue #785: METRON-1230: As a stopgap prior to METRON-777, add more s...
Github user ottobackwards commented on the issue: https://github.com/apache/metron/pull/785 well then. In a world where 777 didn't exist I would be a +1. In world where 777 does exist, i'm a +0, since I feel the effort would have been better spent reviewing 777, but think this is technically good. ---
[GitHub] metron pull request #882: METRON-1380: Create a typosquatting use-case (comm...
Github user asfgit closed the pull request at: https://github.com/apache/metron/pull/882 ---
[GitHub] metron pull request #785: METRON-1230: As a stopgap prior to METRON-777, add...
Github user asfgit closed the pull request at: https://github.com/apache/metron/pull/785 ---
[GitHub] metron issue #882: METRON-1380: Create a typosquatting use-case (commit afte...
Github user justinleet commented on the issue: https://github.com/apache/metron/pull/882 +1, thanks for the hard work in getting everything cleaned up! ---
[GitHub] metron pull request #879: METRON-1378: Create a summarizer
Github user asfgit closed the pull request at: https://github.com/apache/metron/pull/879 ---
[GitHub] metron pull request #899: METRON-1405: Add Boyer-Moore majority vote algorit...
Github user ottobackwards commented on a diff in the pull request: https://github.com/apache/metron/pull/899#discussion_r161821411 --- Diff: metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java --- @@ -220,4 +219,156 @@ public Object apply(List args) { } } } + + @Stellar(namespace = "BOYERMOORE" + , name = "ADD" + , description = "Adds value to a Boyer-Moore list. [Boyer-Moore](https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm)" + , params = { +"state - state holder for list of values. If null, add will initialize a new state value.", +"value(s) - single object or list of values to add to the state object." + } + , returns = "Current state of the Boyer-Moore algorithm representing the current value that" + + "holds a plurality across all values added thus far." + ) + public static class BoyerMooreAdd extends BaseStellarFunction { + +@Override +public Object apply(List args) { + if (args.size() < 2) { +throw new IllegalArgumentException( +"Must pass an initial state (may be null) and at least one value to add to the list"); + } else { +BoyerMooreState state = ConversionUtils.convert(args.get(0), BoyerMooreState.class); +if (state == null) { + state = new BoyerMooreState(); +} +Object secondArg = args.get(1); +if (secondArg instanceof List) { + state.addAll((List) secondArg); +} else { + state.add(secondArg); +} +return state; + } +} + } + + public static class BoyerMooreState { +private Long counter; +private Object m; + +public BoyerMooreState() { + counter = 0L; +} + +public BoyerMooreState(Optional> previousStates, Optional currentState) { + this(); + currentState.ifPresent(boyerMooreState -> { +m = boyerMooreState.getPlurality(); +counter = boyerMooreState.getCounter(); + }); + for (BoyerMooreState state : previousStates.orElse(new ArrayList<>())) { +Object plurality = state.getPlurality(); +Long pluralityCount = state.getCounter(); +add(plurality, pluralityCount); + } +} + +public Object add(Object item) { + if (item != null) { +if (counter == 0) { + m = item; + counter = 1L; +} else if (item.equals(m)) { + counter++; +} else { + counter--; +} + } + return m; +} + +public Object add(Object item, Long counter) { + if (item != null) { +if (this.counter == 0) { + m = item; + this.counter = counter; --- End diff -- So, Stellar will parse and create Numbers, I'm wondering if there is a possibility of strange errors or rounding here. Like there are two not equal numbers but they end up equal through the conversion process. Does that make sense? ---
[GitHub] metron pull request #899: METRON-1405: Add Boyer-Moore majority vote algorit...
Github user ottobackwards commented on a diff in the pull request: https://github.com/apache/metron/pull/899#discussion_r161817448 --- Diff: metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MathFunctions.java --- @@ -220,4 +219,156 @@ public Object apply(List args) { } } } + + @Stellar(namespace = "BOYERMOORE" + , name = "ADD" + , description = "Adds value to a Boyer-Moore list. [Boyer-Moore](https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm)" + , params = { +"state - state holder for list of values. If null, add will initialize a new state value.", +"value(s) - single object or list of values to add to the state object." + } + , returns = "Current state of the Boyer-Moore algorithm representing the current value that" + + "holds a plurality across all values added thus far." + ) + public static class BoyerMooreAdd extends BaseStellarFunction { + +@Override +public Object apply(List args) { + if (args.size() < 2) { +throw new IllegalArgumentException( +"Must pass an initial state (may be null) and at least one value to add to the list"); + } else { +BoyerMooreState state = ConversionUtils.convert(args.get(0), BoyerMooreState.class); +if (state == null) { + state = new BoyerMooreState(); +} +Object secondArg = args.get(1); +if (secondArg instanceof List) { + state.addAll((List) secondArg); +} else { + state.add(secondArg); +} +return state; + } +} + } + + public static class BoyerMooreState { +private Long counter; +private Object m; + +public BoyerMooreState() { + counter = 0L; +} + +public BoyerMooreState(Optional> previousStates, Optional currentState) { + this(); + currentState.ifPresent(boyerMooreState -> { +m = boyerMooreState.getPlurality(); +counter = boyerMooreState.getCounter(); + }); + for (BoyerMooreState state : previousStates.orElse(new ArrayList<>())) { +Object plurality = state.getPlurality(); +Long pluralityCount = state.getCounter(); +add(plurality, pluralityCount); + } +} + +public Object add(Object item) { + if (item != null) { +if (counter == 0) { + m = item; + counter = 1L; +} else if (item.equals(m)) { + counter++; +} else { + counter--; +} + } + return m; +} + +public Object add(Object item, Long counter) { + if (item != null) { +if (this.counter == 0) { + m = item; + this.counter = counter; +} else if (item.equals(m)) { + this.counter += counter; +} else if (counter > this.counter) { + m = item; + this.counter = counter - this.counter; +} else { + this.counter = 0L; +} + } + return m; +} + +public Object getPlurality() { + return m; +} + +public Object addAll(List items) { + for (Object item : items) { +add(item); + } + return m; +} + +public Long getCounter() { + return counter; +} + } + + @Stellar(namespace = "BOYERMOORE" + , name = "PLURALITY" + , description = "Calculates the item with current plurality in a Boyer-Moore list. [Boyer-Moore](https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm)" + , params = { + "state - state holder for list of values. If null, add will initialize a new state value." + } --- End diff -- This is confusing. It is the same description as ADD() but I think that ADD is correct and this should be "Returns the object that has plurality" ? ---
[GitHub] metron issue #879: METRON-1378: Create a summarizer
Github user nickwallen commented on the issue: https://github.com/apache/metron/pull/879 +0 I'm sure what's here is solid, but I have not reviewed it myself. I just want to clear the way for this to get merged. I don't necessarily like the usability of this approach, but I think it is a good first step. And merging this doesn't preclude providing alternative approaches or enhancing this approach for better usability later. I do love the use case that was added that drove the need for this functionality. So good stuff @cestella! ---
[GitHub] metron pull request #786: METRON-1231: Separate Sensor name and topic in the...
Github user asfgit closed the pull request at: https://github.com/apache/metron/pull/786 ---
[GitHub] metron issue #858: METRON-1344: Externalize the infrastructural components u...
Github user ottobackwards commented on the issue: https://github.com/apache/metron/pull/858 @merrimanr I am +1 on getting this down to the feature branch and moving on. Due to the way the feature branch works, I think it is OK to do so if @cestella doesn't get back in time. It's been two weeks. Let's get this moving ---
[GitHub] metron pull request #888: METRON-1389: Zeppelin notebook import does not wor...
Github user ottobackwards commented on a diff in the pull request: https://github.com/apache/metron/pull/888#discussion_r161746734 --- Diff: metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py --- @@ -162,9 +163,21 @@ def zeppelin_notebook_import(self, env): --- End diff -- Do we have to reg ext this out? Can't we just pull it out of the dict? ---
[GitHub] metron pull request #888: METRON-1389: Zeppelin notebook import does not wor...
Github user anandsubbu commented on a diff in the pull request: https://github.com/apache/metron/pull/888#discussion_r161735380 --- Diff: metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py --- @@ -150,12 +151,36 @@ def zeppelin_notebook_import(self, env): env.set_params(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) + +# With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password +# for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details). +ses = requests.session() + +# Check if authentication is enabled on the Zeppelin server +try: +conn = ses.get(ambari_format('http://{zeppelin_server_url}/api/login')) + +# Establish connection if authentication is enabled +try: +# The following credentials are created at install time by Ambari at /etc/zeppelin/conf/shiro.ini +# when Shiro auth is enabled on the Zeppelin server +zeppelin_payload = {'userName': 'admin', 'password' : 'admin'} --- End diff -- Hi @cestella and @ottobackwards - I found a way to get the Shiro INI content from Ambari and parse the admin credentials. Please have a look at my latest commit. ---