[jira] [Updated] (METRON-2284) Metron Profiler for Spark doesn't work as expected

[Maxim D (Jira)]

     [ 
https://issues.apache.org/jira/browse/METRON-2284?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Maxim D updated METRON-2284:
----------------------------
    Description: 
Used command:

{code}
/usr/hdp/current/spark2-client/bin/spark-submit --class 
org.apache.metron.profiler.spark.cli.BatchProfilerCLI --properties-file 
/usr/hcp/current/metron/config/batch-profiler.properties 
~/metron-profiler-spark-0.7.1.1.9.1.0-6.jar --config 
/usr/hcp/current/metron/config/batch-profiler.properties --profiles 
~/profiler.json
{code}

 cat /usr/hcp/current/metron/config/batch-profiler.properties
{code}
profiler.batch.input.path=/tmp/test_data.logs
profiler.batch.input.format=json
profiler.period.duration=15
profiler.period.duration.units=MINUTES
{code}

 
cat ~/profiler.json
{code}
{
 "profiles":[
   {
     "profile":"batchtest5",
     "onlyif":"source.type == 'testsource' and devicehostname == 
'windows9.something.com'",
     "foreach":"devicehostname",
     "init":{
       "val":"SET_INIT()"
     },
     "update":{
       "val":"SET_ADD(val, IS_EMPTY(devicehostname))"
     },
    "result":{
       "profile":"val"
    }
   }
 ],
 "timestampField":"timestamp"
}
{code}

 cat test_data.logs
{code}
{"devicehostname": "windows9.something.com", "timestamp": 1567241981000, 
"source.type": "testsource"}
{code}

Stellar statement
{code}
PROFILE_GET('batchtest5', 'windows9.something.com', PROFILE_FIXED(100, 'DAYS'))
{code}

Returns:
{code}
[[true]]
{code}

Expected result:
{code}
[[false]]
{code}

  was:
Used command:

{code}
/usr/hdp/current/spark2-client/bin/spark-submit --class 
org.apache.metron.profiler.spark.cli.BatchProfilerCLI --properties-file 
/usr/hcp/current/metron/config/batch-profiler.properties 
~/metron-profiler-spark-0.7.1.1.9.1.0-6.jar --config 
/usr/hcp/current/metron/config/batch-profiler.properties --profiles 
~/profiler.json
{code}

 cat /usr/hcp/current/metron/config/batch-profiler.properties
{code}
profiler.batch.input.path=/tmp/test_data.logs
profiler.batch.input.format=json
profiler.period.duration=15
profiler.period.duration.units=MINUTES
{code}

 
cat ~/profiler.json
{code}
{
 "profiles":[
   {
     "profile":"batchtest5",
     "onlyif":"source.type == 'testsource' and devicehostname == 
'windows9.something.com'",
     "foreach":"devicehostname",
     "init":{
       "val":"SET_INIT()"
     },
     "update":{
       "val":"SET_ADD(val, IS_EMPTY(devicehostname))"
     },
    "result":{
       "profile":"val"
    }
   }
 ],
 "timestampField":"timestamp"
}
{code}

 cat test_data.logs
{code}
{"devicehostname": "windows9.something.com", "timestamp": 1567241981000, 
"source.type": "testsource"}
{code}

Stellar statement
PROFILE_GET('batchtest5', 'windows9.something.com', PROFILE_FIXED(100, 'DAYS'))

Returns:

[[true]]

Expected result:

[[false]]


> Metron Profiler for Spark doesn't work as expected
> --------------------------------------------------
>
>                 Key: METRON-2284
>                 URL: https://issues.apache.org/jira/browse/METRON-2284
>             Project: Metron
>          Issue Type: Bug
>            Reporter: Maxim D
>            Priority: Major
>
> Used command:
> {code}
> /usr/hdp/current/spark2-client/bin/spark-submit --class 
> org.apache.metron.profiler.spark.cli.BatchProfilerCLI --properties-file 
> /usr/hcp/current/metron/config/batch-profiler.properties 
> ~/metron-profiler-spark-0.7.1.1.9.1.0-6.jar --config 
> /usr/hcp/current/metron/config/batch-profiler.properties --profiles 
> ~/profiler.json
> {code}
>  cat /usr/hcp/current/metron/config/batch-profiler.properties
> {code}
> profiler.batch.input.path=/tmp/test_data.logs
> profiler.batch.input.format=json
> profiler.period.duration=15
> profiler.period.duration.units=MINUTES
> {code}
>  
> cat ~/profiler.json
> {code}
> {
>  "profiles":[
>    {
>      "profile":"batchtest5",
>      "onlyif":"source.type == 'testsource' and devicehostname == 
> 'windows9.something.com'",
>      "foreach":"devicehostname",
>      "init":{
>        "val":"SET_INIT()"
>      },
>      "update":{
>        "val":"SET_ADD(val, IS_EMPTY(devicehostname))"
>      },
>     "result":{
>        "profile":"val"
>     }
>    }
>  ],
>  "timestampField":"timestamp"
> }
> {code}
>  cat test_data.logs
> {code}
> {"devicehostname": "windows9.something.com", "timestamp": 1567241981000, 
> "source.type": "testsource"}
> {code}
> Stellar statement
> {code}
> PROFILE_GET('batchtest5', 'windows9.something.com', PROFILE_FIXED(100, 
> 'DAYS'))
> {code}
> Returns:
> {code}
> [[true]]
> {code}
> Expected result:
> {code}
> [[false]]
> {code}



--
This message was sent by Atlassian Jira
(v8.3.4#803005)