http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/sidebar.html ---------------------------------------------------------------------- diff --git a/griffin-ui/sidebar.html b/griffin-ui/sidebar.html deleted file mode 100644 index 1a9fdd9..0000000 --- a/griffin-ui/sidebar.html +++ /dev/null @@ -1,84 +0,0 @@ -<!-- - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<div id="rightbar" ng-controller="SideBarCtrl" class="" style="background-color:#262626;"> - - <div id="side-bar-stats" class="row" style="border-bottom:5px solid #060606;margin-left:0;margin-right:0"> - - <!--<div class="row-container">--> - <div class="col-xs-12 col-md-12 col-lg-6 " id="sidebar-option"> - - <div > - - <div class="sidebar-stat-center"> - <img src="/img/sidebar1.png" > - <span><a class="bark-link" href="#/dataassets">{{datasets}} Data Assets</a></span> - </div> - - <div class="sidebar-stat-center"> - <img src="/img/sidebar2.png"> - <span><a class="bark-link" href="#/metrics">{{metrics}} DQ Metrics</a></span> - </div> - - </div> - - </div> - <div class="col-xs-12 col-md-12 col-lg-6"> - <div id="data-asset-pie"></div> - </div> - <!--</div>--> - </div> - - - <div id="side-bar-metrics" class="row" style="margin-top:20px;overflow-y:auto;overflow-x:hidden;margin-left:0;margin-right:0;height:1px;"> - <div ng-repeat="outerItems in briefmetrics"> - <div class="well" style="padding:0px;background:transparent;border:0px;position:relative; "> - <!-- <img class="bullseye" src="/img/bullseye.png" title="image1" > --> - <div class="col-sm-4 col-lg-4 col-md-4 "><h4 ><a href="#/metrics/{{outerItems.name}}"> {{outerItems.name}}</a></h4></div> - <div class="col-sm-3 col-lg-3 col-md-3 " style="display: flex;justify-content: center;height:46px;"><div ng-class="outerItems.dq >=90 ? 'led-green':'led-yellow'" style="align-self: center;"></div></div> - <div class="col-sm-5 col-lg-5 col-md-5 " style="display: flex;justify-content: center;height:46px;"><a style="align-self: center;" class="btn btn-primary" href="https://github.com/eBay/DQSolution/issues"; target="_blank">Report issue</a></div> - </div> - <br/> - <div class="well side-bar-scroll"> - <div class="panel-group" ng-repeat="items in outerItems.metrics" style="margin-bottom:0px;background:transparent;"> - <div class="panel panel-default" style="position:relative;background:transparent;"> - <div class="panel-heading" style="background:transparent;"> - <h4 class="panel-title side-metrics"> - <a data-toggle="collapse" href={{"#"+outerItems.name+$index}} ng-init="items.tag=true" ng-click="items.tag=!items.tag;draw(items, $parent.$index, $index);" > - <i class="faChevron" ng-class="items.tag ? 'fa fa-caret-right':'fa fa-caret-down'" style="width:10px"></i> - - <i class="fa fa-line-chart faArrows"></i> - <span class="side-date">{{items.timestamp | date:'MM/dd HH:mm':'-0700'}}</span> - - <span class="side-name" title="{{items.name}}">{{items.name |strShorten}}</span> - <span ng-if="items.dq <= 100" class="pull-right" ng-class="items.dqfail?'side-percent-red':'side-percent'">{{items.dq | number:2}}%</span> - <span ng-if="items.dq > 100" class="pull-right" ng-class="items.dqfail?'side-percent-red':''">{{items.dq/1000 | number:0}}K</span> - </a> - </h4> - </div> - <div id={{outerItems.name+$index}} class="panel-collapse collapse"> - <!-- <div class="panel-body"> - <highchart config="items.chartId"> - </div> --> - <div class="panel-body" style="cursor:pointer;padding:0px 15px;"> - <div id={{'chart'+$parent.$index+'-'+$index}} class="side-chart"></div> - </div> - </div> - </div> - </div> - </div> - </div> - </div> -</div>
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/karma.conf.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/karma.conf.js b/griffin-ui/tests/ut/karma.conf.js deleted file mode 100644 index 4f8a5e1..0000000 --- a/griffin-ui/tests/ut/karma.conf.js +++ /dev/null @@ -1,102 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -// Karma configuration -// Generated on Thu Apr 07 2016 15:02:00 GMT+0800 (China Standard Time) -//http://monicalent.com/blog/2015/02/11/karma-tests-angular-js-require-j/ - -module.exports = function(config) { - config.set({ - - // base path that will be used to resolve all patterns (eg. files, exclude) - basePath: '../../', - - - // frameworks to use - // available frameworks: https://npmjs.org/browse/keyword/karma-adapter - frameworks: ['jasmine', 'requirejs'], - - - files: [ - 'tests/ut/test-main.js', - {pattern:'bower_components/**/*.js', included:false}, - {pattern:'node_modules/angular-mocks/angular-mocks.js', included:false}, - {pattern: 'js/**/*.js', included: false}, - {pattern: 'tests/**/*spec.js', included: false} - ], - - - // list of files to exclude - exclude: [ - 'js/main.js', - 'js/bs.js', - 'js/routes.js', - 'bower_components/**/*test*/**/*.js', - 'bower_components/**/*spec.js', - // 'node_modules/**/*spec.js', - // 'node_modules/**/*spec*/**/*.js', - // 'node_modules/**/*test.js' - ], - - - // preprocess matching files before serving them to the browser - // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor - preprocessors: { - 'js/**/*.js':['coverage'] - }, - - coverageReporter: { - type:'lcov', - dir: 'tests/ut/test-coverage' - }, - - - // test results reporter to use - // possible values: 'dots', 'progress' - // available reporters: https://npmjs.org/browse/keyword/karma-reporter - // reporters: ['progress'], - reporters: ['mocha', 'coverage', 'progress'], - - - // web server port - port: 9876, - - - // enable / disable colors in the output (reporters and logs) - colors: true, - - - // level of logging - // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG - logLevel: config.LOG_DEBUG, - - - // enable / disable watching file and executing tests whenever any file changes - autoWatch: true, - - - // start these browsers - // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher - browsers: ['Chrome'], - - - // Continuous Integration mode - // if true, Karma captures browsers, runs the tests and exits - singleRun: false, - - // Concurrency level - // how many browser should be started simultaneous - concurrency: Infinity - }) -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/createrule-ac-ctrl.spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/createrule-ac-ctrl.spec.js b/griffin-ui/tests/ut/specs/controllers/createrule-ac-ctrl.spec.js deleted file mode 100644 index bf34dcb..0000000 --- a/griffin-ui/tests/ut/specs/controllers/createrule-ac-ctrl.spec.js +++ /dev/null @@ -1,129 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/controllers/createrule-ac-ctrl'], - function(angular, mocks, CreateRuleACCtrl) { - describe('Test /js/controllers/createrule-ac-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $scope, $rootScope, $controller, $httpBackend, $config, $location, toaster, $timeout, $route; - beforeEach(inject(function(_$rootScope_ , _$controller_, _$httpBackend_, _$config_, _$location_, _$timeout_){ - $rootScope = _$rootScope_; - $controller = _$controller_; - $httpBackend = _$httpBackend_; - $config = _$config_; - $location = _$location_; - $timeout = _$timeout_; - $route = {}; - toaster = {}; - })); - - beforeEach(function(){ - $scope = $rootScope.$new(); - controller = $controller('CreateRuleACCtrl', {$scope: $scope, $route: $route, toaster: toaster }); - }); - - describe("if the controller of CreateRuleACCtrl exists",function(){ - it('controller exists', function(){ - expect(controller).toBeDefined(); - }); - }) - - describe("check if parameters are available",function(){ - - it('if the controller exists', function(){ - expect(controller).toBeDefined(); - }); - - it('$scope.value and $config.value should be right', function(){ - expect($scope.currentStep).toBe(1); - expect(typeof $scope.selection).toEqual("object"); - expect($config.uri.dbtree).toBeTruthy(); - expect($config.uri.schemadefinition).toBeTruthy(); - }); - - it('$scope.ruleTypes should be right', function(){ - expect($scope.ruleTypes).toEqual(['Accuracy', 'Validity', 'Anomaly Detection', 'Publish Metrics']); - }); - - }) - - describe("check if function are work",function(){ - - it('$scope.selectNodeLabelTarget works well', function(){ - expect($scope.selectNodeLabelTarget ).toBeDefined(); - }); - - it('$scope.toggleSelection works well', function(){ - expect($scope.toggleSelection).toBeDefined(); - }); - - it('$scope.toggleAll works well', function(){ - expect($scope.toggleAll).toBeDefined(); - }); - - it('$scope.form is a object', function(){ - expect(typeof $scope.form).toBe("object"); - }); - - }) - - describe("$scope.form test",function(){ - - it('the type of $scope.form', function(){ - expect(typeof $scope.form).toBe("object"); - }); - - it('the type of $scope.form.submit', function(){ - expect(typeof $scope.form.submit).toBe('function'); - }); - - it('the type of $scope.form.save', function(){ - expect(typeof $scope.form.save).toBe('function'); - }); - - }) - - describe("httpGet $config.uri.dbtree test",function(){ - beforeEach(function(){ - $httpBackend.when('GET', $config.uri.dbtree).respond({"age": 16,"name": "li"}); - $httpBackend.when('GET', $config.uri.schemadefinition).respond({"age": 16,"name": "li"}); - }); - - it('http response', function(){ - $httpBackend.flush(); - expect($scope.dbListTarget).toBeTruthy(); - }); - - it('test watch and http request', function(){ - $scope.currentNode = {"name":"ha"}; - $scope.$digest(); - - $httpBackend.flush(); - expect($scope.selection.length).toBe(0); - expect($scope.selectedAll).toBe(false); - - expect($scope.schemaCollection.age).toBe(16); - }); - - afterEach(function() { - $httpBackend.verifyNoOutstandingExpectation(); - $httpBackend.verifyNoOutstandingRequest(); - }); - }) - }); - } -) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/createrule-pu-ctrl.spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/createrule-pu-ctrl.spec.js b/griffin-ui/tests/ut/specs/controllers/createrule-pu-ctrl.spec.js deleted file mode 100644 index 9bf9af2..0000000 --- a/griffin-ui/tests/ut/specs/controllers/createrule-pu-ctrl.spec.js +++ /dev/null @@ -1,105 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/controllers/createrule-pu-ctrl'], - function(angular, mocks, CreateRulePUCtrl) { - describe('Test /js/controllers/createrule-pu-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $scope, $rootScope, $controller, $httpBackend, $config, $location, toaster, $timeout, $route; - beforeEach(inject(function(_$rootScope_ , _$controller_, _$httpBackend_, _$config_, _$location_, _$timeout_){ - $rootScope = _$rootScope_; - $controller = _$controller_; - $httpBackend = _$httpBackend_; - $config = _$config_; - $location = _$location_; - $timeout = _$timeout_; - $route = {}; - toaster = ['error', 'Error', "hello", "lisan"];/*test data*/ - })); - - beforeEach(function(){ - $scope = $rootScope.$new(); - controller = $controller('CreateRulePUCtrl', {$scope: $scope, $route: $route, toaster: toaster }); - }); - - describe("if the controller of CreateRulePUCtrl exists",function(){ - it('controller exists', function(){ - expect(controller).toBeDefined(); - }); - }) - - describe("$scope.value",function(){ - - it('$scope.currentStep == 1', function(){ - expect($scope.currentStep).toBe(1); - }); - - it('$scope.scheduleTypes', function(){ - expect($scope.scheduleTypes).toEqual(['Daily', 'Weekly', 'Monthly', 'Hourly']); - }); - - }) - - describe("$scope.form test",function(){ - - it('the type of $scope.form', function(){ - expect(typeof $scope.form).toBe("object"); - }); - - it('the type of $scope.form.submit', function(){ - expect(typeof $scope.form.submit).toBe('function'); - }); - - it('the type of $scope.form.save', function(){ - expect(typeof $scope.form.save).toBe('function'); - }); - - }) - - /*describe("function errorMessage() test",function(){ - // test function - it('errorMessage', function(){ - var message = ["hello"]; - var i = 2; - var msg = true; - $scope.errorMessage(i,msg); - expect(toaster).toEqual(['error', 'Error',"hello"]); - }); - - })*/ - - describe('Testing $watch expressions', function() { - it('test using $digest', function() { - var form = {}; - form.basic = {"name": "li"}; - form.publishUrl = "publishUrl"; - $scope.form = form; - form.basic.name = "ha"; - $scope.$digest(); - form.basic.name = "la"; - $scope.$digest(); - - expect($scope.form.publishUrl).toEqual('http://dq.vip.ebay.com/api/v1/publishmetric/la'); - - }); - }); - - - - }); - } -) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/createrule-va-ctrl.spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/createrule-va-ctrl.spec.js b/griffin-ui/tests/ut/specs/controllers/createrule-va-ctrl.spec.js deleted file mode 100644 index 990ef4e..0000000 --- a/griffin-ui/tests/ut/specs/controllers/createrule-va-ctrl.spec.js +++ /dev/null @@ -1,110 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/controllers/createrule-va-ctrl'], - function(angular, mocks, CreateRuleVACtrl) { - describe('Test /js/controllers/createrule-va-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $scope, $rootScope, $controller, $httpBackend, $config, $location, toaster, $timeout, $route; - beforeEach(inject(function(_$rootScope_ , _$controller_, _$httpBackend_, _$config_, _$location_, _$timeout_){ - $rootScope = _$rootScope_; - $controller = _$controller_; - $httpBackend = _$httpBackend_; - $config = _$config_; - $location = _$location_; - $timeout = _$timeout_; - $route = {}; - toaster = {}; - })); - - beforeEach(function(){ - $scope = $rootScope.$new(); - controller = $controller('CreateRuleVACtrl', {$scope: $scope, $route: $route, toaster: toaster }); - }); - - describe("if the controller of CreateRuleVACtrl exists",function(){ - it('controller exists', function(){ - expect(controller).toBeDefined(); - }); - }) - - describe("check if parameters are available",function(){ - - it('$scope.value and $config.value should be right', function(){ - expect($scope.currentStep).toBe(1); - expect($config.uri.dbtree).toBeTruthy(); - expect($config.uri.schemadefinition).toBeTruthy(); - }); - - it('$scope.ruleTypes should be right', function(){ - expect($scope.ruleTypes).toEqual(['Accuracy', 'Validity', 'Anomaly Detection', 'Publish Metrics']); - }); - - }) - - describe("check if form function are work",function(){ - - it('$scope.form is a object', function(){ - expect(typeof $scope.form).toBe("object"); - }); - - }) - - describe("$scope.form test",function(){ - - it('the type of $scope.form', function(){ - expect(typeof $scope.form).toBe("object"); - }); - - it('the type of $scope.form.submit', function(){ - expect(typeof $scope.form.submit).toBe('function'); - }); - - it('the type of $scope.form.save', function(){ - expect(typeof $scope.form.save).toBe('function'); - }); - - }) - - describe("httpGet $config.uri.dbtree test",function(){ - beforeEach(function(){ - $httpBackend.when('GET', $config.uri.dbtree).respond({"age": 16,"name": "li"}); - $httpBackend.when('GET', $config.uri.schemadefinition).respond({"age": 15,"name": "wei"}); - }); - - it('http response', function(){ - $httpBackend.flush(); - expect($scope.dbList).toBeTruthy(); - }); - - it('test watch and http request', function(){ - $scope.currentNode = {"name":"ha"}; - $scope.$digest(); - - $httpBackend.flush(); - expect($scope.schemaCollection.age).toBe(15); - }); - - afterEach(function() { - $httpBackend.verifyNoOutstandingExpectation(); - $httpBackend.verifyNoOutstandingRequest(); - }); - }) - - }); - } -) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/createrule0-ctrl.spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/createrule0-ctrl.spec.js b/griffin-ui/tests/ut/specs/controllers/createrule0-ctrl.spec.js deleted file mode 100644 index 8df4af1..0000000 --- a/griffin-ui/tests/ut/specs/controllers/createrule0-ctrl.spec.js +++ /dev/null @@ -1,62 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/controllers/createrule0-ctrl'], - function(angular, mocks, CreateRule0Ctrl) { - describe('Test /js/controllers/createrule0-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $scope, $rootScope, $controller, $httpBackend, $config, $location, toaster, $timeout; - - beforeEach(inject(function(_$rootScope_ , _$controller_, _$httpBackend_, _$config_, _$location_, _$timeout_){ - $rootScope = _$rootScope_; - $controller = _$controller_; - $httpBackend = _$httpBackend_; - $config = _$config_; - $location = _$location_; - $timeout = _$timeout_; - toaster = {}; - })); - - beforeEach(function(){ - $scope = $rootScope.$new(); - controller = $controller('CreateRule0Ctrl', {$scope: $scope, toaster: toaster }); - }); - - describe("if the controller of CreateRule0Ctrl exists",function(){ - it('controller exists', function(){ - expect(controller).toBeDefined(); - }); - }) - - describe("if the $scope.click exists",function(){ - - it('$scope.click', function(){ - expect($scope.click).toBeDefined(); - }); - - it('should change location when setting it via click function', inject(function() { - var url = '/index'; - $scope.click(url); - spyOn($location, 'path').and.returnValue(url); - })); - - }) - - - }); - } -) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/health-ctrl.spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/health-ctrl.spec.js b/griffin-ui/tests/ut/specs/controllers/health-ctrl.spec.js deleted file mode 100644 index b35e40a..0000000 --- a/griffin-ui/tests/ut/specs/controllers/health-ctrl.spec.js +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['require', 'angular', 'angularMocks', 'js/controllers/health-ctrl', 'echarts'], - function(require, angular, mocks, HealthCtrl) { - describe('Test /js/controllers/health-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $scope, $rootScope, $controller, $httpBackend, $config, $location, toaster, $timeout, $route; - - beforeEach(inject(function(_$rootScope_ , _$controller_, _$httpBackend_, _$config_, _$location_, _$timeout_){ - $rootScope = _$rootScope_; - $controller = _$controller_; - $httpBackend = _$httpBackend_; - $config = _$config_; - $location = _$location_; - $timeout = _$timeout_; - toaster = {}; - $route = {}; - })); - - beforeEach(function(){ - $scope = $rootScope.$new(); - controller = $controller('HealthCtrl', {$scope: $scope, $route: $route, toaster: toaster }); - }); - - describe("if the controller of HealthCtrl exists",function(){ - it('controller exists', function(){ - expect(controller).toBeDefined(); - }); - }) - - describe("check if parameters are available",function(){ - - it('$scope.value and $config.value should be right', function(){ - expect($config.uri.heatmap).toBeTruthy(); - }); - - }) - - describe("httpGet $config.uri.dbtree test",function(){ - beforeEach(function(){ - $httpBackend.when('GET', $config.uri.heatmap).respond([{"name":"unknown","dq":0.0,"metrics":[{"name":"mean","dq":4835.3,"dqfail":0,"timestamp":1470387389994,"metricType":"","assetName":null,"details":[]},{"name":"test1001","dq":1638.6,"dqfail":0,"timestamp":1470387312289,"metricType":"","assetName":null,"details":[]},{"name":"test_publish","dq":99.8,"dqfail":0,"timestamp":1463994766925,"metricType":"","assetName":null,"details":[]},{"name":"v","dq":99.8,"dqfail":0,"timestamp":1463994766925,"metricType":"","assetName":null,"details":[]}]},{"name":"Hadoop","dq":0.0,"metrics":[{"name":"movie_acc","dq":97.0,"dqfail":0,"timestamp":1470009600000,"metricType":"","assetName":null,"details":[]},{"name":"movie_acc_test2","dq":97.0,"dqfail":0,"timestamp":1470009600000,"metricType":"","assetName":null,"details":[]},{"name":"hadoop_accuracy_1","dq":99.053,"dqfail":0,"timestamp":1467356400000,"metricType":"","assetName":null,"details":[]}]},{"name":"Bullseye","dq":0.0,"metrics":[{"na me":"test_accuracy_1","dq":98.952,"dqfail":1,"timestamp":1467439200000,"metricType":"","assetName":null,"details":[]},{"name":"test_accuracy_2","dq":99.103,"dqfail":0,"timestamp":1467439200000,"metricType":"","assetName":null,"details":[]},{"name":"TotalCount_asset1","dq":5056215.0,"dqfail":0,"timestamp":1467439200000,"metricType":"","assetName":null,"details":[]},{"name":"TotalCount_asset2","dq":1.229703E7,"dqfail":0,"timestamp":1467356400000,"metricType":"","assetName":null,"details":[]},{"name":"aw","dq":5056215.0,"dqfail":0,"timestamp":1467439200000,"metricType":"Bollinger","assetName":null,"details":[]}]}]); - $httpBackend.flush(); - }); - - it('http response', function(){ - // expect($scope.dbList).toBeTruthy(); - }); - - afterEach(function() { - $httpBackend.verifyNoOutstandingExpectation(); - $httpBackend.verifyNoOutstandingRequest(); - }); - }) - - - }); - } -) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/metrics-ctrl.spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/metrics-ctrl.spec.js b/griffin-ui/tests/ut/specs/controllers/metrics-ctrl.spec.js deleted file mode 100644 index 0036101..0000000 --- a/griffin-ui/tests/ut/specs/controllers/metrics-ctrl.spec.js +++ /dev/null @@ -1,76 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/controllers/metrics-ctrl'], - function(angular, mocks, MetricsCtrl) { - describe('Test /js/controllers/metrics-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $rootScope, $controller, $httpBackend, $config, $filter, $routeParams, $timeout, $compile, $route; - beforeEach(inject(function(_$rootScope_, _$controller_, _$httpBackend_, _$config_, _$filter_, _$timeout_, _$compile_){ - $rootScope = _$rootScope_; - $controller = _$controller_; - $httpBackend = _$httpBackend_; - $config = _$config_; - $filter = _$filter_; - $routeParams = {}; - // $routeParams.siteId = 'abc'; - $timeout = _$timeout_; - $compile = _$compile_; - $route = {}; - })); - - describe("function test",function(){ - var $scope; - - beforeEach(function(){ - $scope = $rootScope.$new(); - controller = $controller('MetricsCtrl', {$scope: $scope, $route: $route, $routeParams: $routeParams}); - }); - - it('controller exists', function(){ - // var controller = $controller('MetricsCtrl', {$scope: $scope, $routeParams: $routeParams}); - expect(controller).toBeDefined(); - }); - - it('$config.uri.dashboard', function(){ - expect($config.uri.dashboard).toBeTruthy(); - }); - - it('$scope.showBig works well', function(){ - expect($scope.showBig).toBeDefined(); - }); - - describe("http test",function(){ - beforeEach(function(){ - $httpBackend.when('GET', $config.uri.dashboard).respond({"age": 16,"name": "li"}); - $httpBackend.flush(); - }); - - it('http response', function(){ - expect($scope.dashboard.age).toBe(16); - }); - - afterEach(function() { - $httpBackend.verifyNoOutstandingExpectation(); - $httpBackend.verifyNoOutstandingRequest(); - }); - }) - - }) - }); - } -) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/nav-ctrl_spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/nav-ctrl_spec.js b/griffin-ui/tests/ut/specs/controllers/nav-ctrl_spec.js deleted file mode 100644 index b4549fb..0000000 --- a/griffin-ui/tests/ut/specs/controllers/nav-ctrl_spec.js +++ /dev/null @@ -1,47 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/controllers/nav-ctrl'], - function(angular, mocks, navCtrl) { - describe('Test /js/controllers/nav-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $controller, $config, $httpBackend, $location; - - beforeEach(inject(function(_$controller_, _$config_, _$httpBackend_, _$location_){ - $controller = _$controller_; - $config = _$config_; - $httpBackend = _$httpBackend_; - $location = _$location_; - })); - - describe('$scope functions are set properly', function(){ - var $scope; - - beforeEach(function(){ - $scope = {}; - $controller('NavCtrl', {$scope:$scope}); - }) - - it('$scope.isActive is defined correctly', function(){ - $location.path('/home'); - expect($scope.isActive('/home')).toBeTruthy(); - }); - - }); - }); - } -); http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/rule-ctrl.spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/rule-ctrl.spec.js b/griffin-ui/tests/ut/specs/controllers/rule-ctrl.spec.js deleted file mode 100644 index c5b0df8..0000000 --- a/griffin-ui/tests/ut/specs/controllers/rule-ctrl.spec.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/controllers/rule-ctrl'], - function(angular, mocks, RuleCtrl) { - describe('Test /js/controllers/rule-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $scope, $rootScope, $controller, $httpBackend, $config, $location, toaster, $timeout, $route; - - beforeEach(inject(function(_$rootScope_ , _$controller_, _$httpBackend_, _$config_, _$location_, _$timeout_){ - $rootScope = _$rootScope_; - $controller = _$controller_; - $httpBackend = _$httpBackend_; - $config = _$config_; - $location = _$location_; - $timeout = _$timeout_; - toaster = {}; - $route = {}; - })); - - beforeEach(function(){ - $scope = $rootScope.$new(); - controller = $controller('RuleCtrl', {$scope: $scope, $route: $route, toaster: toaster }); - }); - - describe("if the controller of RuleCtrl exists",function(){ - it('controller exists', function(){ - expect(controller).toBeDefined(); - }); - }) - - describe("check if createRowCollection",function(){ - - it('createRowCollection', function(){ - expect($scope.rowCollection).not.toEqual([]); - }); - - }) - - - }); - } -) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/sidebar-ctrl_spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/sidebar-ctrl_spec.js b/griffin-ui/tests/ut/specs/controllers/sidebar-ctrl_spec.js deleted file mode 100644 index 5b5a949..0000000 --- a/griffin-ui/tests/ut/specs/controllers/sidebar-ctrl_spec.js +++ /dev/null @@ -1,89 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/controllers/sidebar-ctrl'], - function(angular, mocks, sidebarCtrl) { - describe('Test /js/controllers/sidebar-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $controller, $httpBackend, $config, $filter, $timeout, $compile; - beforeEach(inject(function(_$controller_, _$httpBackend_, _$config_, _$filter_, _$timeout_, _$compile_){ - $controller = _$controller_; - $httpBackend = _$httpBackend_; - $config = _$config_; - $filter = _$filter_; - $timeout = _$timeout_; - $compile = _$compile_; - })); - - describe("http unit test",function(){ - var $scope; - beforeEach(function(){ - $scope = {}; - $controller('SideBarCtrl', {$scope:$scope}); - }) - it('$scope.str works well', function(){ - // expect($scope.str).toEqual("hello"); - }); - it('url works well', function(){ - expect($config.uri.statistics).toBeTruthy(); - }); - it('function resizeSidebar() works well', function(){ - // expect($scope.resizeSidebar).toBeDefined(); - }); - describe("http get test",function(){ - beforeEach(function(){ - $httpBackend.when('GET', $config.uri.statistics).respond( - { - "assets": 16, - "metrics": 20, - "status":{ - "health": 16, - "warn": 3, - "invalid": 1 - } - } - ); - - $httpBackend.when('GET', $config.uri.briefmetrics).respond( - { - "assets": 16, - "metrics": 20, - "status":{ - "health": 16, - "warn": 3, - "invalid": 1 - } - } - ); - $httpBackend.flush(); - }); - - it("$scope.chartConfig is defined", function(){ - expect($scope.chartConfig).toBeDefined(); - }); - - afterEach(function() { - $httpBackend.verifyNoOutstandingExpectation(); - $httpBackend.verifyNoOutstandingRequest(); - }); - }) - - - }) - }); - } -) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/controllers/viewrule-ctrl.spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/controllers/viewrule-ctrl.spec.js b/griffin-ui/tests/ut/specs/controllers/viewrule-ctrl.spec.js deleted file mode 100644 index 1aa3c61..0000000 --- a/griffin-ui/tests/ut/specs/controllers/viewrule-ctrl.spec.js +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/controllers/viewrule-ctrl'], - function(angular, mocks, ViewRuleCtrl) { - describe('Test /js/controllers/viewrule-ctrl.js', function(){ - beforeEach(function(){ - module('app.controllers'); - module('app.services'); - }); - var $scope, $rootScope, $controller, $filter, $httpBackend, $config, $location, toaster, $timeout, $compile; - - beforeEach(inject(function(_$rootScope_ , _$controller_, _$filter_, _$httpBackend_, _$config_, _$location_, _$timeout_, _$compile_){ - $rootScope = _$rootScope_; - $controller = _$controller_; - $filter = _$filter_; - $httpBackend = _$httpBackend_; - $config = _$config_; - $location = _$location_; - $timeout = _$timeout_; - toaster = {}; - $compile = _$compile_; - })); - - beforeEach(function(){ - $scope = $rootScope.$new(); - controller = $controller('ViewRuleCtrl', {$scope: $scope, toaster: toaster}); - }); - - describe("if the controller of ViewRuleCtrl exists",function(){ - it('controller exists', function(){ - expect(controller).toBeDefined(); - }); - }) - - describe("check if parameters are available",function(){ - it('$scope.value and $config.value should be right', function(){ - expect($config.uri.rulemetric).toBeTruthy(); - }); - }) - - describe("httpGet $config.uri.rulemetric test",function(){ - beforeEach(function(){ - $httpBackend.when('GET', $config.uri.rulemetric).respond({"age": 16,"name": "li"}); - $httpBackend.flush(); - }); - - it('http response', function(){ - // expect($scope.dbList).toBeTruthy(); - }); - - afterEach(function() { - $httpBackend.verifyNoOutstandingExpectation(); - $httpBackend.verifyNoOutstandingRequest(); - }); - }) - - - }); - } -) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/filters/strmap_spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/filters/strmap_spec.js b/griffin-ui/tests/ut/specs/filters/strmap_spec.js deleted file mode 100644 index a2bbee0..0000000 --- a/griffin-ui/tests/ut/specs/filters/strmap_spec.js +++ /dev/null @@ -1,33 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/filters/strmap'], - function(angular, mocks, filter) { - - describe('Test /js/filters/strmap.js', function() { - beforeEach(module('app.filters')); - - // Our first test!!!! - it('map correctly for index', mocks.inject(function($filter) { - // console.log($filter('strShorten')('1234567890123444444')); - var arr = ['Apple', 'Orange', 'Pale']; - expect($filter('strmap')(0, arr)).toEqual(arr[0]); - expect($filter('strmap')(3, arr)).toEqual(3); - - })); - - - }); - } -); http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/filters/strshorten_spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/filters/strshorten_spec.js b/griffin-ui/tests/ut/specs/filters/strshorten_spec.js deleted file mode 100644 index e6ffa7d..0000000 --- a/griffin-ui/tests/ut/specs/filters/strshorten_spec.js +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/filters/strshorten'], - function(angular, mocks, filter) { - - describe('Test /js/filters/strshorten.js', function() { - beforeEach(module('app.filters')); - // Here we register the function returned by the myFilter AMD module - // beforeEach(mocks.module(function($filterProvider) { - // $filterprovider.register('stringsFilter', filter); - // })); - - // Our first test!!!! - it('string should be shorten', mocks.inject(function($filter) { - // console.log($filter('strShorten')('1234567890123444444')); - expect($filter('strShorten')('abcde')).toEqual('abcde'); - expect($filter('strShorten')('1234567890123444444')).toEqual('123456789012...'); - })); - - // it('should be true', function(){ - // expect(1).not.toBeNull(); - // }); - - }); - } -); http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/specs/services/services_spec.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/specs/services/services_spec.js b/griffin-ui/tests/ut/specs/services/services_spec.js deleted file mode 100644 index 8709dc1..0000000 --- a/griffin-ui/tests/ut/specs/services/services_spec.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -define(['angular', 'angularMocks', 'js/services/services'], - function(angular, mocks, filter) { - - describe('Test /js/services/services.js', function() { - beforeEach(module('app.services')); - // Here we register the function returned by the myFilter AMD module - // beforeEach(mocks.module(function($filterProvider) { - // $filterprovider.register('stringsFilter', filter); - // })); - - // Our first test!!!! - it('$config uri is set', mocks.inject(function($config) { - // expect($config).not.toBeNull(); - expect(Object.getOwnPropertyNames($config.uri).length).toBeGreaterThan(0); - })); - - // it('should be true', function(){ - // expect(1).not.toBeNull(); - // }); - - }); - } -); http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-ui/tests/ut/test-main.js ---------------------------------------------------------------------- diff --git a/griffin-ui/tests/ut/test-main.js b/griffin-ui/tests/ut/test-main.js deleted file mode 100644 index ef3c9bb..0000000 --- a/griffin-ui/tests/ut/test-main.js +++ /dev/null @@ -1,97 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -//for demo purpose -var allTestFiles = []; -var TEST_REGEXP = /(spec|test)\.js$/i; - -// Get a list of all the test files to include -Object.keys(window.__karma__.files).forEach(function(file) { - if (TEST_REGEXP.test(file)) { - // Normalize paths to RequireJS module names. - // If you require sub-dependencies of test files to be loaded as-is (requiring file extension) - // then do not normalize the paths - var normalizedTestModule = file.replace(/^\/base\/|\.js$/g, ''); - // var normalizedTestModule = file.replace(/^\/base\//, '').replace(/\.js$/, ''); - // console.log(normalizedTestModule); - allTestFiles.push(normalizedTestModule); - } -}); - -require.config({ - // Karma serves files under /base, which is the basePath from your config file - baseUrl: '/base', - - waitSeconds: 200, - // dynamically load all test files - deps: allTestFiles, - // dpes: ['C:/temp/Bark_UI/tests/ut/specs/my_first_spec'], - - - // we have to kickoff jasmine, as it is asynchronous - callback: window.__karma__.start, - paths: { - // 'domReady': '../bower_components/domReady/domReady', - 'angular': '/base/bower_components/angular/angular', - 'angularMocks': '/base/node_modules/angular-mocks/angular-mocks', - 'angularRoute': '/base/bower_components/angular-route/angular-route', - - 'ngAnimate': '/base/bower_components/angular-animate/angular-animate', - 'ngToaster': '/base/bower_components/AngularJS-Toaster/toaster', - - 'jquery': '/base/bower_components/jquery/dist/jquery', - 'bootstrap': '/base/bower_components/bootstrap/dist/js/bootstrap', - - - 'echarts': '/base/bower_components/echarts/dist/echarts', - }, - shim: { - 'angular': { - deps: ['jquery'], - exports: 'angular' - }, - // 'angularMocks': { - // exports: 'angularMocks', - // deps: ['angular'] - // }, - 'angularMocks': {deps: ['angular'], 'exports': 'angular.mock'}, - 'angularRoute': { - deps: ['angular'], - exports: 'angularRoute' - }, - 'ngSmartTable': { - deps: ['angular'], - exports: 'ngSmartTable' - }, - 'ngAnimate': { - deps: ['angular'], - exports: 'ngAnimate' - }, - 'ngToaster': { - deps: ['angular', 'ngAnimate'], - exports: 'ngToaster' - }, - - 'jquery': { - exports: 'jquery' - }, - 'bootstrap': { - exports: 'bootstrap', - deps: ['jquery'] - }, - 'echarts': { - exports: 'echarts' - } - } -}); http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/pom.xml ---------------------------------------------------------------------- diff --git a/measure/measure-batch/pom.xml b/measure/measure-batch/pom.xml new file mode 100644 index 0000000..ddb398b --- /dev/null +++ b/measure/measure-batch/pom.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>measure</artifactId> + <groupId>org.apache.griffin</groupId> + <version>0.1.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>measure-batch</artifactId> + <packaging>jar</packaging> + + <name>Apache Griffin :: Measures :: Measure Batch</name> + <url>http://maven.apache.org</url> + + +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/resources/config-old.json ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/resources/config-old.json b/measure/measure-batch/src/main/resources/config-old.json new file mode 100644 index 0000000..63dee69 --- /dev/null +++ b/measure/measure-batch/src/main/resources/config-old.json @@ -0,0 +1,45 @@ +{ + "name": "accu1", + "type": "accuracy", + + "source": { + "connector": { + "type": "hive", + "version": "1.2", + "config": { + "table.name": "users_info_src", + "partitions": "dt=20170410, hour=14" + } + } + }, + + "target": { + "connector": { + "type": "hive", + "version": "1.2", + "config": { + "database": "default", + "table.name": "users_info_target", + "partitions": "dt=20170410, hour=14; dt=20170410, hour=15" + } + } + }, + + "evaluateRule": { + "sampleRatio": 1, + "assertion": { + "type": "DSL-griffin", + "rules": [ + { + "rule": "@Key ${source}['user_id'] === ${target}['user_id']" + }, + { + "rule": "${source}['first_name'] === ${target}['first_name']; ${source}['last_name'] === ${target}['last_name']; ${source}['address'] === ${target}['address']" + }, + { + "rule": "${source}['email'] === ${target}['email']; ${source}['phone'] === ${target}['phone']; ${source}['post_code'] === ${target}['post_code']" + } + ] + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/resources/config.json ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/resources/config.json b/measure/measure-batch/src/main/resources/config.json new file mode 100644 index 0000000..edd2e6a --- /dev/null +++ b/measure/measure-batch/src/main/resources/config.json @@ -0,0 +1,29 @@ +{ + "name": "accu1", + "type": "accuracy", + + "source": { + "type": "hive", + "version": "1.2", + "config": { + "database": "default", + "table.name": "users_info_src", + "partitions": "dt=23123, hour=432; dt=35464, hour=4657" + } + }, + + "target": { + "type": "hive", + "version": "1.2", + "config": { + "database": "default", + "table.name": "users_info_target", + "partitions": "dt=23123, hour=432; dt=35464, hour=4657" + } + }, + + "evaluateRule": { + "sampleRatio": 0.2, + "rules": "$source.user_id = $target.user_id AND $source.first_name = $target.first_name AND $source.last_name = $target.last_name AND $source.address = $target.address AND $source.email = $target.email AND $source.phone = $target.phone AND $source.post_code = $target.post_code" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/resources/env.json ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/resources/env.json b/measure/measure-batch/src/main/resources/env.json new file mode 100644 index 0000000..2e72601 --- /dev/null +++ b/measure/measure-batch/src/main/resources/env.json @@ -0,0 +1,29 @@ +{ + "spark": { + "log.level": "INFO", + "checkpoint.dir": "hdfs:///griffin/batch/cp", + "config": {} + }, + + "persist": [ + { + "type": "hdfs", + "config": { + "path": "hdfs:///griffin/streaming/persist", + "max.persist.lines": 10000, + "max.lines.per.file": 10000 + } + }, + { + "type": "http", + "config": { + "method": "post", + "api": "http://phxbark4dq-360935.stratus.phx.ebay.com:8080/"; + } + } + ], + + "cleaner": { + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/resources/log4j.properties b/measure/measure-batch/src/main/resources/log4j.properties new file mode 100644 index 0000000..bd31e15 --- /dev/null +++ b/measure/measure-batch/src/main/resources/log4j.properties @@ -0,0 +1,5 @@ +log4j.rootLogger=INFO, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%c] - %m%n \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/Application.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/Application.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/Application.scala new file mode 100644 index 0000000..6fe6bcb --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/Application.scala @@ -0,0 +1,87 @@ +package org.apache.griffin.measure.batch + +import org.apache.griffin.measure.batch.algo.{Algo, BatchAccuracyAlgo} +import org.apache.griffin.measure.batch.config.params._ +import org.apache.griffin.measure.batch.config.params.env._ +import org.apache.griffin.measure.batch.config.params.user._ +import org.apache.griffin.measure.batch.config.reader._ +import org.apache.griffin.measure.batch.config.validator.AllParamValidator +import org.apache.griffin.measure.batch.log.Loggable + +import scala.util.{Failure, Success, Try} + +object Application extends Loggable { + + def main(args: Array[String]): Unit = { + info(args.toString) + if (args.length < 2) { + error("Usage: class <env-param> <user-param> [List of String split by comma: raw | local | hdfs(default)]") + sys.exit(-1) + } + + val envParamFile = args(0) + val userParamFile = args(1) + val (envFsType, userFsType) = if (args.length > 2) { + val fsTypes = args(2).trim.split(",") + if (fsTypes.length == 1) (fsTypes(0).trim, fsTypes(0).trim) + else if (fsTypes.length >= 2) (fsTypes(0).trim, fsTypes(1).trim) + else ("hdfs", "hdfs") + } else ("hdfs", "hdfs") + + info(envParamFile) + info(userParamFile) + + // read param files + val envParam = readParamFile[EnvParam](envParamFile, envFsType) match { + case Success(p) => p + case Failure(ex) => { + error(ex.getMessage) + sys.exit(-2) + } + } + val userParam = readParamFile[UserParam](userParamFile, userFsType) match { + case Success(p) => p + case Failure(ex) => { + error(ex.getMessage) + sys.exit(-2) + } + } + val allParam: AllParam = AllParam(envParam, userParam) + + // validate param files + validateParams(allParam) match { + case Failure(ex) => { + error(ex.getMessage) + sys.exit(-3) + } + case _ => { + info("params validation pass") + } + } + + // choose algorithm // fixme: not done, need to choose algorithm by param + val dqType = allParam.userParam.dqType + val algo: Algo = BatchAccuracyAlgo(allParam) + + algo.run match { + case Failure(ex) => { + error(ex.getMessage) + sys.exit(-4) + } + case _ => { + info("calculation finished") + } + } + } + + private def readParamFile[T <: Param](file: String, fsType: String)(implicit m : Manifest[T]): Try[T] = { + val paramReader = ParamReaderFactory.getParamReader(file, fsType) + paramReader.readConfig[T] + } + + private def validateParams(allParam: AllParam): Try[Boolean] = { + val allParamValidator = AllParamValidator() + allParamValidator.validate(allParam) + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/AccuracyAlgo.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/AccuracyAlgo.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/AccuracyAlgo.scala new file mode 100644 index 0000000..d22add2 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/AccuracyAlgo.scala @@ -0,0 +1,6 @@ +package org.apache.griffin.measure.batch.algo + + +trait AccuracyAlgo extends Algo { + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/Algo.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/Algo.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/Algo.scala new file mode 100644 index 0000000..d7047ee --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/Algo.scala @@ -0,0 +1,16 @@ +package org.apache.griffin.measure.batch.algo + +import org.apache.griffin.measure.batch.config.params.env._ +import org.apache.griffin.measure.batch.config.params.user._ +import org.apache.griffin.measure.batch.log.Loggable + +import scala.util.Try + +trait Algo extends Loggable with Serializable { + + val envParam: EnvParam + val userParam: UserParam + + def run(): Try[_] + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgo.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgo.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgo.scala new file mode 100644 index 0000000..e4ed616 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgo.scala @@ -0,0 +1,166 @@ +package org.apache.griffin.measure.batch.algo + +import java.util.Date + +import org.apache.griffin.measure.batch.algo.core.AccuracyCore +import org.apache.griffin.measure.batch.config.params.AllParam +import org.apache.griffin.measure.batch.connector._ +import org.apache.griffin.measure.batch.rule._ +import org.apache.griffin.measure.batch.rule.expr._ +import org.apache.griffin.measure.batch.persist._ +import org.apache.griffin.measure.batch.result._ +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.hive.HiveContext +import org.apache.spark.{SparkConf, SparkContext} + +import scala.util.{Failure, Success, Try} + + +case class BatchAccuracyAlgo(allParam: AllParam) extends AccuracyAlgo { + val envParam = allParam.envParam + val userParam = allParam.userParam + + def run(): Try[_] = { + Try { + val metricName = userParam.name + + val conf = new SparkConf().setAppName(metricName) + val sc = new SparkContext(conf) + val sqlContext = new HiveContext(sc) + + // start time + val startTime = new Date().getTime() + + // get persists + val persist: Persist = PersistFactory(envParam.persistParams, metricName).getPersists(startTime) + + // get spark application id + val applicationId = sc.applicationId + + // start + persist.start(applicationId) + + // rules + val ruleFactory = RuleFactory(userParam.evaluateRuleParam) + val rule: StatementExpr = ruleFactory.generateRule() + val ruleAnalyzer: RuleAnalyzer = RuleAnalyzer(rule) + + // global cache data + val globalCachedData = CacheDataUtil.genCachedMap(None, ruleAnalyzer.globalCacheExprs, Map[String, Any]()) + val globalFinalCachedData = CacheDataUtil.filterCachedMap(ruleAnalyzer.globalFinalCacheExprs, globalCachedData) + + // data connector + val sourceDataConnector: DataConnector = + DataConnectorFactory.getDataConnector(sqlContext, userParam.sourceParam, + ruleAnalyzer.sourceGroupbyExprs, ruleAnalyzer.sourceCacheExprs, + ruleAnalyzer.sourceFinalCacheExprs, globalFinalCachedData, + ruleAnalyzer.whenClauseExpr + ) match { + case Success(cntr) => { + if (cntr.available) cntr + else throw new Exception("source data connection error!") + } + case Failure(ex) => throw ex + } + val targetDataConnector: DataConnector = + DataConnectorFactory.getDataConnector(sqlContext, userParam.targetParam, + ruleAnalyzer.targetGroupbyExprs, ruleAnalyzer.targetCacheExprs, + ruleAnalyzer.targetFinalCacheExprs, globalFinalCachedData, + ruleAnalyzer.whenClauseExpr + ) match { + case Success(cntr) => { + if (cntr.available) cntr + else throw new Exception("target data connection error!") + } + case Failure(ex) => throw ex + } + + // get metadata +// val sourceMetaData: Iterable[(String, String)] = sourceDataConnector.metaData() match { +// case Success(md) => md +// case _ => throw new Exception("source metadata error!") +// } +// val targetMetaData: Iterable[(String, String)] = targetDataConnector.metaData() match { +// case Success(md) => md +// case _ => throw new Exception("target metadata error!") +// } + + // get data + val sourceData: RDD[(Product, Map[String, Any])] = sourceDataConnector.data() match { + case Success(dt) => dt + case Failure(ex) => throw ex + } + val targetData: RDD[(Product, Map[String, Any])] = targetDataConnector.data() match { + case Success(dt) => dt + case Failure(ex) => throw ex + } + + // accuracy algorithm + val (accuResult, missingRdd, matchingRdd) = accuracy(sourceData, targetData, ruleAnalyzer) + + // end time + val endTime = new Date().getTime + persist.log(endTime, s"calculation using time: ${endTime - startTime} ms") + + // persist result + persist.result(endTime, accuResult) + val missingRecords = missingRdd.map(record2String(_, ruleAnalyzer.sourcePersistExprs, ruleAnalyzer.targetPersistExprs)) + persist.missRecords(missingRecords) + + // persist end time + val persistEndTime = new Date().getTime + persist.log(persistEndTime, s"persist using time: ${persistEndTime - endTime} ms") + + // finish + persist.finish() + + // context stop + sc.stop + + } + } + + def wrapInitData(data: Map[String, Any]): (Map[String, Any], Map[String, Any]) = { + (data, Map[String, Any]()) + } + + def accuracy(sourceData: RDD[(Product, Map[String, Any])], targetData: RDD[(Product, Map[String, Any])], ruleAnalyzer: RuleAnalyzer + ): (AccuracyResult, RDD[(Product, (Map[String, Any], Map[String, Any]))], RDD[(Product, (Map[String, Any], Map[String, Any]))]) = { + + // 1. wrap data + val sourceWrappedData: RDD[(Product, (Map[String, Any], Map[String, Any]))] = sourceData.map(r => (r._1, wrapInitData(r._2))) + val targetWrappedData: RDD[(Product, (Map[String, Any], Map[String, Any]))] = targetData.map(r => (r._1, wrapInitData(r._2))) + + // 2. cogroup + val allKvs = sourceWrappedData.cogroup(targetWrappedData) + + // 3. accuracy calculation + val (accuResult, missingRdd, matchingRdd) = AccuracyCore.accuracy(allKvs, ruleAnalyzer) + + (accuResult, missingRdd, matchingRdd) + } + + def record2String(rec: (Product, (Map[String, Any], Map[String, Any])), sourcePersist: Iterable[Expr], targetPersist: Iterable[Expr]): String = { + val (key, (data, info)) = rec + val persistData = getPersistMap(data, sourcePersist) + val persistInfo = info.mapValues { value => + value match { + case vd: Map[String, Any] => getPersistMap(vd, targetPersist) + case v => v + } + } + s"${persistData} [${persistInfo}]" + } + + private def getPersistMap(data: Map[String, Any], persist: Iterable[Expr]): Map[String, Any] = { + val persistMap = persist.map(e => (e._id, e.desc)).toMap + data.flatMap { pair => + val (k, v) = pair + persistMap.get(k) match { + case Some(d) => Some((d -> v)) + case _ => None + } + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/core/AccuracyCore.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/core/AccuracyCore.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/core/AccuracyCore.scala new file mode 100644 index 0000000..639dd8a --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/core/AccuracyCore.scala @@ -0,0 +1,78 @@ +package org.apache.griffin.measure.batch.algo.core + +import org.apache.griffin.measure.batch.rule.RuleAnalyzer +import org.apache.griffin.measure.batch.result._ +import org.apache.spark.rdd.RDD + + +object AccuracyCore { + + type V = Map[String, Any] + type T = Map[String, Any] + + def accuracy(allKvs: RDD[(Product, (Iterable[(V, T)], Iterable[(V, T)]))], ruleAnalyzer: RuleAnalyzer + ): (AccuracyResult, RDD[(Product, (V, T))], RDD[(Product, (V, T))]) = { + val result: RDD[(Long, Long, List[(Product, (V, T))], List[(Product, (V, T))])] = allKvs.map { kv => + val (key, (sourceDatas, targetDatas)) = kv + + // result: (missCount, matchCount, missDataList, matchDataList) + val rslt = sourceDatas.foldLeft((0L, 0L, List[(Product, (V, T))](), List[(Product, (V, T))]())) { (sr, sourcePair) => + val matchResult = if (targetDatas.isEmpty) { + (false, Map[String, Any](MismatchInfo.wrap("no target"))) + } else { + targetDatas.foldLeft((false, Map[String, Any]())) { (tr, targetPair) => + if (tr._1) tr + else matchData(sourcePair, targetPair, ruleAnalyzer) + } + } + + if (matchResult._1) { + val matchItem = (key, sourcePair) + (sr._1, sr._2 + 1, sr._3, sr._4 :+ matchItem) + } else { + val missItem = (key, (sourcePair._1, sourcePair._2 ++ matchResult._2)) + (sr._1 + 1, sr._2, sr._3 :+ missItem, sr._4) + } + } + + rslt + } + + val missRdd = result.flatMap(_._3) + val matchRdd = result.flatMap(_._4) + + def seq(cnt: (Long, Long), rcd: (Long, Long, Any, Any)): (Long, Long) = { + (cnt._1 + rcd._1, cnt._2 + rcd._2) + } + def comb(c1: (Long, Long), c2: (Long, Long)): (Long, Long) = { + (c1._1 + c2._1, c1._2 + c2._2) + } + val countPair = result.aggregate((0L, 0L))(seq, comb) + + (AccuracyResult(countPair._1, (countPair._1 + countPair._2)), missRdd, matchRdd) + } + + private def matchData(source: (V, T), target: (V, T), ruleAnalyzer: RuleAnalyzer): (Boolean, T) = { + + // 1. merge source and target cached data + val mergedData: Map[String, Any] = mergeData(source, target) + + // 2. check valid + if (ruleAnalyzer.rule.valid(mergedData)) { + // 3. substitute the cached data into statement, get the statement value + // currently we can not get the mismatch reason, we need to add such information to figure out how it mismatches + ((ruleAnalyzer.rule.calculate(mergedData) match { + case Some(b: Boolean) => b + case _ => false + }), Map[String, Any](MismatchInfo.wrap("not matched"), TargetInfo.wrap(target._1))) + } else { + (false, Map[String, Any](MismatchInfo.wrap("invalid to compare"), TargetInfo.wrap(target._1))) + } + + } + + private def mergeData(source: (V, T), target: (V, T)): Map[String, Any] = { + source._1 ++ target._1 + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/AllParam.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/AllParam.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/AllParam.scala new file mode 100644 index 0000000..b5618a8 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/AllParam.scala @@ -0,0 +1,14 @@ +package org.apache.griffin.measure.batch.config.params + +import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty} +import com.fasterxml.jackson.annotation.JsonInclude.Include +import org.apache.griffin.measure.batch.config.params.env._ +import org.apache.griffin.measure.batch.config.params.user._ + +// simply composite of env and user params, for convenient usage +@JsonInclude(Include.NON_NULL) +case class AllParam( @JsonProperty("env") envParam: EnvParam, + @JsonProperty("user") userParam: UserParam + ) extends Param { + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/Param.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/Param.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/Param.scala new file mode 100644 index 0000000..ce94c5c --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/Param.scala @@ -0,0 +1,7 @@ +package org.apache.griffin.measure.batch.config.params + +trait Param extends Serializable { + + def validate(): Boolean = true + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/CleanerParam.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/CleanerParam.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/CleanerParam.scala new file mode 100644 index 0000000..a29004c --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/CleanerParam.scala @@ -0,0 +1,10 @@ +package org.apache.griffin.measure.batch.config.params.env + +import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty} +import com.fasterxml.jackson.annotation.JsonInclude.Include +import org.apache.griffin.measure.batch.config.params.Param + +@JsonInclude(Include.NON_NULL) +case class CleanerParam() extends Param { + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/EnvParam.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/EnvParam.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/EnvParam.scala new file mode 100644 index 0000000..79e2575 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/EnvParam.scala @@ -0,0 +1,13 @@ +package org.apache.griffin.measure.batch.config.params.env + +import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty} +import com.fasterxml.jackson.annotation.JsonInclude.Include +import org.apache.griffin.measure.batch.config.params.Param + +@JsonInclude(Include.NON_NULL) +case class EnvParam( @JsonProperty("spark") sparkParam: SparkParam, + @JsonProperty("persist") persistParams: List[PersistParam], + @JsonProperty("cleaner") cleanerParam: CleanerParam + ) extends Param { + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/PersistParam.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/PersistParam.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/PersistParam.scala new file mode 100644 index 0000000..655f19b --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/PersistParam.scala @@ -0,0 +1,12 @@ +package org.apache.griffin.measure.batch.config.params.env + +import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty} +import com.fasterxml.jackson.annotation.JsonInclude.Include +import org.apache.griffin.measure.batch.config.params.Param + +@JsonInclude(Include.NON_NULL) +case class PersistParam( @JsonProperty("type") persistType: String, + @JsonProperty("config") config: Map[String, Any] + ) extends Param { + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/SparkParam.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/SparkParam.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/SparkParam.scala new file mode 100644 index 0000000..ba6f9a6 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/SparkParam.scala @@ -0,0 +1,13 @@ +package org.apache.griffin.measure.batch.config.params.env + +import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty} +import com.fasterxml.jackson.annotation.JsonInclude.Include +import org.apache.griffin.measure.batch.config.params.Param + +@JsonInclude(Include.NON_NULL) +case class SparkParam( @JsonProperty("log.level") logLevel: String, + @JsonProperty("checkpoint.dir") cpDir: String, + @JsonProperty("config") config: Map[String, Any] + ) extends Param { + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/DataConnectorParam.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/DataConnectorParam.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/DataConnectorParam.scala new file mode 100644 index 0000000..7a8b2b2 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/DataConnectorParam.scala @@ -0,0 +1,13 @@ +package org.apache.griffin.measure.batch.config.params.user + +import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty} +import com.fasterxml.jackson.annotation.JsonInclude.Include +import org.apache.griffin.measure.batch.config.params.Param + +@JsonInclude(Include.NON_NULL) +case class DataConnectorParam( @JsonProperty("type") conType: String, + @JsonProperty("version") version: String, + @JsonProperty("config") config: Map[String, Any] + ) extends Param { + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/EvaluateRuleParam.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/EvaluateRuleParam.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/EvaluateRuleParam.scala new file mode 100644 index 0000000..edf0a38 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/EvaluateRuleParam.scala @@ -0,0 +1,12 @@ +package org.apache.griffin.measure.batch.config.params.user + +import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty} +import com.fasterxml.jackson.annotation.JsonInclude.Include +import org.apache.griffin.measure.batch.config.params.Param + +@JsonInclude(Include.NON_NULL) +case class EvaluateRuleParam( @JsonProperty("sampleRatio") sampleRatio: Double, + @JsonProperty("rules") rules: String + ) extends Param { + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/UserParam.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/UserParam.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/UserParam.scala new file mode 100644 index 0000000..be0d3c8 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/UserParam.scala @@ -0,0 +1,15 @@ +package org.apache.griffin.measure.batch.config.params.user + +import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty} +import com.fasterxml.jackson.annotation.JsonInclude.Include +import org.apache.griffin.measure.batch.config.params.Param + +@JsonInclude(Include.NON_NULL) +case class UserParam(@JsonProperty("name") name: String, + @JsonProperty("type") dqType: String, + @JsonProperty("source") sourceParam: DataConnectorParam, + @JsonProperty("target") targetParam: DataConnectorParam, + @JsonProperty("evaluateRule") evaluateRuleParam: EvaluateRuleParam + ) extends Param { + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReader.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReader.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReader.scala new file mode 100644 index 0000000..e30513f --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReader.scala @@ -0,0 +1,20 @@ +package org.apache.griffin.measure.batch.config.reader + +import org.apache.griffin.measure.batch.config.params.Param +import org.apache.griffin.measure.batch.utils.JsonUtil + +import scala.util.Try + +case class ParamFileReader(file: String) extends ParamReader { + + def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = { + Try { + val source = scala.io.Source.fromFile(file) + val lines = source.mkString + val param = JsonUtil.fromJson[T](lines) + source.close + param + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamHdfsFileReader.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamHdfsFileReader.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamHdfsFileReader.scala new file mode 100644 index 0000000..473b428 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamHdfsFileReader.scala @@ -0,0 +1,20 @@ +package org.apache.griffin.measure.batch.config.reader + +import org.apache.griffin.measure.batch.config.params.Param +import org.apache.griffin.measure.batch.utils.JsonUtil +import org.apache.griffin.measure.batch.utils.HdfsUtil + +import scala.util.Try + +case class ParamHdfsFileReader(filePath: String) extends ParamReader { + + def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = { + Try { + val source = HdfsUtil.openFile(filePath) + val param = JsonUtil.fromJson[T](source) + source.close + param + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamRawStringReader.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamRawStringReader.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamRawStringReader.scala new file mode 100644 index 0000000..1e931de --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamRawStringReader.scala @@ -0,0 +1,17 @@ +package org.apache.griffin.measure.batch.config.reader + +import org.apache.griffin.measure.batch.config.params.Param +import org.apache.griffin.measure.batch.utils.JsonUtil + +import scala.util.Try + +case class ParamRawStringReader(rawString: String) extends ParamReader { + + def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = { + Try { + val param = JsonUtil.fromJson[T](rawString) + param + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReader.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReader.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReader.scala new file mode 100644 index 0000000..2f42b09 --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReader.scala @@ -0,0 +1,12 @@ +package org.apache.griffin.measure.batch.config.reader + +import org.apache.griffin.measure.batch.log.Loggable +import org.apache.griffin.measure.batch.config.params.Param + +import scala.util.Try + +trait ParamReader extends Loggable with Serializable { + + def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReaderFactory.scala ---------------------------------------------------------------------- diff --git a/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReaderFactory.scala b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReaderFactory.scala new file mode 100644 index 0000000..dd77e8c --- /dev/null +++ b/measure/measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReaderFactory.scala @@ -0,0 +1,22 @@ +package org.apache.griffin.measure.batch.config.reader + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileSystem, Path} + + +object ParamReaderFactory { + + val RawStringRegex = """^(?i)raw$""".r + val LocalFsRegex = """^(?i)local$""".r + val HdfsFsRegex = """^(?i)hdfs$""".r + + def getParamReader(filePath: String, fsType: String): ParamReader = { + fsType match { + case RawStringRegex() => ParamRawStringReader(filePath) + case LocalFsRegex() => ParamFileReader(filePath) + case HdfsFsRegex() => ParamHdfsFileReader(filePath) + case _ => ParamHdfsFileReader(filePath) + } + } + +}
