Repository: hadoop Updated Branches: refs/heads/trunk d060cbea4 -> 62f817d32
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/objectstore-service/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/objectstore-service/pom.xml b/hadoop-ozone/objectstore-service/pom.xml index b20f1e2..ce8fe4c 100644 --- a/hadoop-ozone/objectstore-service/pom.xml +++ b/hadoop-ozone/objectstore-service/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop Ozone Object Store REST Service</name> <packaging>jar</packaging> - <properties> - <hadoop.component>ozone</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 3efb33b..a3f69f6 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop Ozone Manager Server</name> <packaging>jar</packaging> - <properties> - <hadoop.component>ozone</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> <dependency> @@ -63,28 +58,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>copy web resources</id> - <phase>compile</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <target> - <copy toDir="${project.build.directory}/webapps"> - <fileset dir="${basedir}/src/main/webapps"> - <exclude name="**/proto-web.xml"/> - </fileset> - </copy> - </target> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <executions> <execution> @@ -93,19 +66,20 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <goals> <goal>unpack</goal> </goals> + <configuration> + <artifactItems> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdds-server-framework</artifactId> + <outputDirectory>${project.build.outputDirectory} + </outputDirectory> + <includes>webapps/static/**/*.*</includes> + </artifactItem> + </artifactItems> + <overWriteSnapshots>true</overWriteSnapshots> + </configuration> </execution> </executions> - <configuration> - <artifactItems> - <artifactItem> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdds-server-framework</artifactId> - <outputDirectory>${project.build.directory}/</outputDirectory> - <includes>webapps/static/**/*.*</includes> - </artifactItem> - </artifactItems> - <overWriteSnapshots>true</overWriteSnapshots> - </configuration> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html new file mode 100644 index 0000000..ba54cb2 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html @@ -0,0 +1,70 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <meta name="description" content="Ozone Manager"> + + <title>Ozone Manager</title> + + <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet"> + <link href="static/hadoop.css" rel="stylesheet"> + <link href="static/nvd3-1.8.5.min.css" rel="stylesheet"> + + <link href="static/ozone.css" rel="stylesheet"> + +</head> + +<body ng-app="ozoneManager"> + +<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav"> + <div class="container-fluid"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" + aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="#">Ozone Manager</a> + </div> + <navmenu + metrics="{ 'OM metrics' : '#!/metrics/ozoneManager', 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu> + </div> +</header> + +<div class="container-fluid"> + <ng-view></ng-view> +</div><!-- /.container --> + +<script src="static/jquery-3.3.1.min.js"></script> +<script src="static/angular-1.6.4.min.js"></script> +<script src="static/angular-route-1.6.4.min.js"></script> +<script src="static/d3-3.5.17.min.js"></script> +<script src="static/nvd3-1.8.5.min.js"></script> +<script src="static/angular-nvd3-1.0.9.min.js"></script> +<script src="static/ozone.js"></script> +<script src="ozoneManager.js"></script> +<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css new file mode 100644 index 0000000..e442adc --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +body { + padding-top: 50px; +} +.starter-template { + padding: 40px 15px; + text-align: center; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html new file mode 100644 index 0000000..0821899 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html @@ -0,0 +1,18 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<overview> +</overview> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html new file mode 100644 index 0000000..15fba2f --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html @@ -0,0 +1,44 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<h1>OzoneManager Metrics</h1> + +<div ng-repeat="(type,numbers) in $ctrl.metrics.nums"> + <h2>{{type}}</h2> + <div class="container"> + <div class="col-md-6"> + <h3>Requests ({{numbers.ops}} ops)</h3> + <nvd3 options="$ctrl.graphOptions" + data="numbers.all"></nvd3> + </div> + <div class="col-md-6"> + <h3>Failures</h3> + <nvd3 options="$ctrl.graphOptions" + data="numbers.failures"></nvd3> + </div> + </div> +</div> + +<div ng-show="$ctrl.metrics.others.length > 0"> + <h2>Other JMX properties</h2> + + <table class="table"> + <tr ng-repeat="metric in $ctrl.metrics.others"> + <td>{{metric.key}}</td> + <td>{{metric.value}}</td> + </tr> + </table> +</div> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js new file mode 100644 index 0000000..ca03554 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +(function () { + "use strict"; + + var isIgnoredJmxKeys = function (key) { + return key == 'name' || key == 'modelerType' || key.match(/tag.*/); + }; + + angular.module('ozoneManager', ['ozone', 'nvd3']); + angular.module('ozoneManager').config(function ($routeProvider) { + $routeProvider + .when("/metrics/ozoneManager", { + template: "<om-metrics></om-metrics>" + }); + }); + angular.module('ozoneManager').component('omMetrics', { + templateUrl: 'om-metrics.html', + controller: function ($http) { + var ctrl = this; + + ctrl.graphOptions = { + chart: { + type: 'pieChart', + height: 500, + x: function (d) { + return d.key; + }, + y: function (d) { + return d.value; + }, + showLabels: true, + labelType: 'value', + duration: 500, + labelThreshold: 0.01, + valueFormat: function(d) { + return d3.format('d')(d); + }, + legend: { + margin: { + top: 5, + right: 35, + bottom: 5, + left: 0 + } + } + } + }; + + + $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics") + .then(function (result) { + + var groupedMetrics = {others: [], nums: {}}; + var metrics = result.data.beans[0] + for (var key in metrics) { + var numericalStatistic = key.match(/Num([A-Z][a-z]+)(.+?)(Fails)?$/); + if (numericalStatistic) { + var type = numericalStatistic[1]; + var name = numericalStatistic[2]; + var failed = numericalStatistic[3]; + groupedMetrics.nums[type] = groupedMetrics.nums[type] || { + failures: [], + all: [] + }; + if (failed) { + groupedMetrics.nums[type].failures.push({ + key: name, + value: metrics[key] + }) + } else { + if (name == "Ops") { + groupedMetrics.nums[type].ops = metrics[key] + } else { + groupedMetrics.nums[type].all.push({ + key: name, + value: metrics[key] + }) + } + } + } else if (isIgnoredJmxKeys(key)) { + //ignore + } else { + groupedMetrics.others.push({ + 'key': key, + 'value': metrics[key] + }); + } + } + ctrl.metrics = groupedMetrics; + }) + } + }); + +})(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html deleted file mode 100644 index ba54cb2..0000000 --- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html +++ /dev/null @@ -1,70 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<html lang="en"> -<head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - <meta name="description" content="Ozone Manager"> - - <title>Ozone Manager</title> - - <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet"> - <link href="static/hadoop.css" rel="stylesheet"> - <link href="static/nvd3-1.8.5.min.css" rel="stylesheet"> - - <link href="static/ozone.css" rel="stylesheet"> - -</head> - -<body ng-app="ozoneManager"> - -<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav"> - <div class="container-fluid"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" - aria-expanded="false" aria-controls="navbar"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="#">Ozone Manager</a> - </div> - <navmenu - metrics="{ 'OM metrics' : '#!/metrics/ozoneManager', 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu> - </div> -</header> - -<div class="container-fluid"> - <ng-view></ng-view> -</div><!-- /.container --> - -<script src="static/jquery-3.3.1.min.js"></script> -<script src="static/angular-1.6.4.min.js"></script> -<script src="static/angular-route-1.6.4.min.js"></script> -<script src="static/d3-3.5.17.min.js"></script> -<script src="static/nvd3-1.8.5.min.js"></script> -<script src="static/angular-nvd3-1.0.9.min.js"></script> -<script src="static/ozone.js"></script> -<script src="ozoneManager.js"></script> -<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script> -</body> -</html> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css deleted file mode 100644 index e442adc..0000000 --- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -body { - padding-top: 50px; -} -.starter-template { - padding: 40px 15px; - text-align: center; -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html deleted file mode 100644 index 0821899..0000000 --- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html +++ /dev/null @@ -1,18 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<overview> -</overview> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html deleted file mode 100644 index 15fba2f..0000000 --- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html +++ /dev/null @@ -1,44 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<h1>OzoneManager Metrics</h1> - -<div ng-repeat="(type,numbers) in $ctrl.metrics.nums"> - <h2>{{type}}</h2> - <div class="container"> - <div class="col-md-6"> - <h3>Requests ({{numbers.ops}} ops)</h3> - <nvd3 options="$ctrl.graphOptions" - data="numbers.all"></nvd3> - </div> - <div class="col-md-6"> - <h3>Failures</h3> - <nvd3 options="$ctrl.graphOptions" - data="numbers.failures"></nvd3> - </div> - </div> -</div> - -<div ng-show="$ctrl.metrics.others.length > 0"> - <h2>Other JMX properties</h2> - - <table class="table"> - <tr ng-repeat="metric in $ctrl.metrics.others"> - <td>{{metric.key}}</td> - <td>{{metric.value}}</td> - </tr> - </table> -</div> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js deleted file mode 100644 index ca03554..0000000 --- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -(function () { - "use strict"; - - var isIgnoredJmxKeys = function (key) { - return key == 'name' || key == 'modelerType' || key.match(/tag.*/); - }; - - angular.module('ozoneManager', ['ozone', 'nvd3']); - angular.module('ozoneManager').config(function ($routeProvider) { - $routeProvider - .when("/metrics/ozoneManager", { - template: "<om-metrics></om-metrics>" - }); - }); - angular.module('ozoneManager').component('omMetrics', { - templateUrl: 'om-metrics.html', - controller: function ($http) { - var ctrl = this; - - ctrl.graphOptions = { - chart: { - type: 'pieChart', - height: 500, - x: function (d) { - return d.key; - }, - y: function (d) { - return d.value; - }, - showLabels: true, - labelType: 'value', - duration: 500, - labelThreshold: 0.01, - valueFormat: function(d) { - return d3.format('d')(d); - }, - legend: { - margin: { - top: 5, - right: 35, - bottom: 5, - left: 0 - } - } - } - }; - - - $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics") - .then(function (result) { - - var groupedMetrics = {others: [], nums: {}}; - var metrics = result.data.beans[0] - for (var key in metrics) { - var numericalStatistic = key.match(/Num([A-Z][a-z]+)(.+?)(Fails)?$/); - if (numericalStatistic) { - var type = numericalStatistic[1]; - var name = numericalStatistic[2]; - var failed = numericalStatistic[3]; - groupedMetrics.nums[type] = groupedMetrics.nums[type] || { - failures: [], - all: [] - }; - if (failed) { - groupedMetrics.nums[type].failures.push({ - key: name, - value: metrics[key] - }) - } else { - if (name == "Ops") { - groupedMetrics.nums[type].ops = metrics[key] - } else { - groupedMetrics.nums[type].all.push({ - key: name, - value: metrics[key] - }) - } - } - } else if (isIgnoredJmxKeys(key)) { - //ignore - } else { - groupedMetrics.others.push({ - 'key': key, - 'value': metrics[key] - }); - } - } - ctrl.metrics = groupedMetrics; - }) - } - }); - -})(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozonefs/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml index c45aacd..8174b74 100644 --- a/hadoop-ozone/ozonefs/pom.xml +++ b/hadoop-ozone/ozonefs/pom.xml @@ -28,8 +28,6 @@ <properties> <file.encoding>UTF-8</file.encoding> <downloadSources>true</downloadSources> - <hadoop.component>ozone</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> </properties> <build> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index c73be60..8d3e161 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -19,9 +19,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-project-dist</artifactId> + <artifactId>hadoop-project</artifactId> <version>3.2.0-SNAPSHOT</version> - <relativePath>../hadoop-project-dist</relativePath> + <relativePath>../hadoop-project</relativePath> </parent> <artifactId>hadoop-ozone</artifactId> <version>0.3.0-SNAPSHOT</version> @@ -42,6 +42,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <module>ozonefs</module> <module>integration-test</module> <module>objectstore-service</module> + <module>datanode</module> + <module>dist</module> <module>docs</module> </modules> @@ -161,6 +163,25 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <excludeFilterFile combine.self="override"></excludeFilterFile> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>add-classpath-descriptor</id> + <phase>package</phase> + <goals> + <goal>build-classpath</goal> + </goals> + <configuration> + <attach>true</attach> + <prefix>$HDDS_LIB_JARS_DIR</prefix> + <outputFilterFile>true</outputFilterFile> + <includeScope>runtime</includeScope> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index eeec595..b04bbf1 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -28,21 +28,33 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop Ozone Tools</name> <packaging>jar</packaging> - <properties> - <hadoop.component>ozone</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-ozone-common</artifactId> - <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-ozone-client</artifactId> - <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-filesystem</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs-client</artifactId> + <scope>compile</scope> </dependency> <dependency> <groupId>io.dropwizard.metrics</groupId> @@ -59,6 +71,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <artifactId>jmh-generator-annprocess</artifactId> <version>1.19</version> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-integration-test</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> </dependencies> <build> <plugins> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java new file mode 100644 index 0000000..a2df50d --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.freon; + +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests Freon, with MiniOzoneCluster and validate data. + */ +public class TestDataValidate { + + private static MiniOzoneCluster cluster; + private static OzoneConfiguration conf; + + /** + * Create a MiniDFSCluster for testing. + * <p> + * Ozone is made active by setting OZONE_ENABLED = true + * + */ + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(5).build(); + cluster.waitForClusterToBeReady(); + } + + /** + * Shutdown MiniDFSCluster. + */ + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void ratisTestLargeKey() throws Exception { + RandomKeyGenerator randomKeyGenerator = + new RandomKeyGenerator((OzoneConfiguration) cluster.getConf()); + randomKeyGenerator.setNumOfVolumes(1); + randomKeyGenerator.setNumOfBuckets(1); + randomKeyGenerator.setNumOfKeys(1); + randomKeyGenerator.setType(ReplicationType.RATIS); + randomKeyGenerator.setFactor(ReplicationFactor.THREE); + randomKeyGenerator.setKeySize(20971520); + randomKeyGenerator.setValidateWrites(true); + randomKeyGenerator.call(); + Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated()); + Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated()); + Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded()); + Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount()); + } + + @Test + public void standaloneTestLargeKey() throws Exception { + RandomKeyGenerator randomKeyGenerator = + new RandomKeyGenerator((OzoneConfiguration) cluster.getConf()); + randomKeyGenerator.setNumOfVolumes(1); + randomKeyGenerator.setNumOfBuckets(1); + randomKeyGenerator.setNumOfKeys(1); + randomKeyGenerator.setKeySize(20971520); + randomKeyGenerator.setValidateWrites(true); + randomKeyGenerator.call(); + Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated()); + Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated()); + Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded()); + Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount()); + } + + @Test + public void validateWriteTest() throws Exception { + RandomKeyGenerator randomKeyGenerator = + new RandomKeyGenerator((OzoneConfiguration) cluster.getConf()); + randomKeyGenerator.setNumOfVolumes(2); + randomKeyGenerator.setNumOfBuckets(5); + randomKeyGenerator.setNumOfKeys(10); + randomKeyGenerator.setValidateWrites(true); + randomKeyGenerator.call(); + Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated()); + Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated()); + Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded()); + Assert.assertTrue(randomKeyGenerator.getValidateWrites()); + Assert.assertNotEquals(0, randomKeyGenerator.getTotalKeysValidated()); + Assert.assertNotEquals(0, randomKeyGenerator + .getSuccessfulValidationCount()); + Assert.assertEquals(0, randomKeyGenerator + .getUnsuccessfulValidationCount()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java new file mode 100644 index 0000000..d21d399 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.freon; + +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests Freon, with MiniOzoneCluster. + */ +public class TestRandomKeyGenerator { + + private static MiniOzoneCluster cluster; + private static OzoneConfiguration conf; + + /** + * Create a MiniDFSCluster for testing. + * <p> + * Ozone is made active by setting OZONE_ENABLED = true + * + */ + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); + cluster.waitForClusterToBeReady(); + } + + /** + * Shutdown MiniDFSCluster. + */ + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void defaultTest() throws Exception { + RandomKeyGenerator randomKeyGenerator = + new RandomKeyGenerator((OzoneConfiguration) cluster.getConf()); + randomKeyGenerator.setNumOfVolumes(2); + randomKeyGenerator.setNumOfBuckets(5); + randomKeyGenerator.setNumOfKeys(10); + randomKeyGenerator.call(); + Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated()); + Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated()); + Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded()); + Assert.assertEquals(10240 - 36, randomKeyGenerator.getKeyValueLength()); + } + + @Test + public void multiThread() throws Exception { + RandomKeyGenerator randomKeyGenerator = + new RandomKeyGenerator((OzoneConfiguration) cluster.getConf()); + randomKeyGenerator.setNumOfVolumes(10); + randomKeyGenerator.setNumOfBuckets(1); + randomKeyGenerator.setNumOfKeys(10); + randomKeyGenerator.setNumOfThreads(10); + randomKeyGenerator.setKeySize(10240); + randomKeyGenerator.call(); + Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated()); + Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated()); + Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded()); + } + + @Test + public void ratisTest3() throws Exception { + RandomKeyGenerator randomKeyGenerator = + new RandomKeyGenerator((OzoneConfiguration) cluster.getConf()); + randomKeyGenerator.setNumOfVolumes(10); + randomKeyGenerator.setNumOfBuckets(1); + randomKeyGenerator.setNumOfKeys(10); + randomKeyGenerator.setNumOfThreads(10); + randomKeyGenerator.setKeySize(10240); + randomKeyGenerator.setFactor(ReplicationFactor.THREE); + randomKeyGenerator.setType(ReplicationType.RATIS); + randomKeyGenerator.call(); + Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated()); + Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated()); + Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java new file mode 100644 index 0000000..a3ff6c8 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java @@ -0,0 +1,285 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om; + +import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.scm.cli.SQLCLI; +import org.apache.hadoop.ozone.web.handlers.BucketArgs; +import org.apache.hadoop.ozone.web.handlers.KeyArgs; +import org.apache.hadoop.ozone.web.handlers.UserArgs; +import org.apache.hadoop.ozone.web.handlers.VolumeArgs; +import org.apache.hadoop.ozone.web.interfaces.StorageHandler; +import org.apache.hadoop.ozone.web.utils.OzoneUtils; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * This class tests the CLI that transforms om.db into SQLite DB files. + */ +@RunWith(Parameterized.class) +public class TestOmSQLCli { + private MiniOzoneCluster cluster = null; + private StorageHandler storageHandler; + private UserArgs userArgs; + private OzoneConfiguration conf; + private SQLCLI cli; + + private String userName = "userTest"; + private String adminName = "adminTest"; + private String volumeName0 = "volumeTest0"; + private String volumeName1 = "volumeTest1"; + private String bucketName0 = "bucketTest0"; + private String bucketName1 = "bucketTest1"; + private String bucketName2 = "bucketTest2"; + private String keyName0 = "key0"; + private String keyName1 = "key1"; + private String keyName2 = "key2"; + private String keyName3 = "key3"; + + @Parameterized.Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + // Uncomment the below line if we support leveldb in future. + //{OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB} + }); + } + + private String metaStoreType; + + public TestOmSQLCli(String type) { + metaStoreType = type; + } + + /** + * Create a MiniDFSCluster for testing. + * <p> + * Ozone is made active by setting OZONE_ENABLED = true + * + * @throws IOException + */ + @Before + public void setup() throws Exception { + conf = new OzoneConfiguration(); + cluster = MiniOzoneCluster.newBuilder(conf).build(); + cluster.waitForClusterToBeReady(); + storageHandler = new ObjectStoreHandler(conf).getStorageHandler(); + userArgs = new UserArgs(null, OzoneUtils.getRequestID(), + null, null, null, null); + cluster.waitForClusterToBeReady(); + + VolumeArgs createVolumeArgs0 = new VolumeArgs(volumeName0, userArgs); + createVolumeArgs0.setUserName(userName); + createVolumeArgs0.setAdminName(adminName); + storageHandler.createVolume(createVolumeArgs0); + VolumeArgs createVolumeArgs1 = new VolumeArgs(volumeName1, userArgs); + createVolumeArgs1.setUserName(userName); + createVolumeArgs1.setAdminName(adminName); + storageHandler.createVolume(createVolumeArgs1); + + BucketArgs bucketArgs0 = new BucketArgs(volumeName0, bucketName0, userArgs); + storageHandler.createBucket(bucketArgs0); + BucketArgs bucketArgs1 = new BucketArgs(volumeName1, bucketName1, userArgs); + storageHandler.createBucket(bucketArgs1); + BucketArgs bucketArgs2 = new BucketArgs(volumeName0, bucketName2, userArgs); + storageHandler.createBucket(bucketArgs2); + + KeyArgs keyArgs0 = + new KeyArgs(volumeName0, bucketName0, keyName0, userArgs); + keyArgs0.setSize(100); + KeyArgs keyArgs1 = + new KeyArgs(volumeName1, bucketName1, keyName1, userArgs); + keyArgs1.setSize(200); + KeyArgs keyArgs2 = + new KeyArgs(volumeName0, bucketName2, keyName2, userArgs); + keyArgs2.setSize(300); + KeyArgs keyArgs3 = + new KeyArgs(volumeName0, bucketName2, keyName3, userArgs); + keyArgs3.setSize(400); + + OutputStream stream = storageHandler.newKeyWriter(keyArgs0); + stream.close(); + stream = storageHandler.newKeyWriter(keyArgs1); + stream.close(); + stream = storageHandler.newKeyWriter(keyArgs2); + stream.close(); + stream = storageHandler.newKeyWriter(keyArgs3); + stream.close(); + + cluster.getOzoneManager().stop(); + cluster.getStorageContainerManager().stop(); + conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType); + cli = new SQLCLI(conf); + } + + @After + public void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + // After HDDS-357, we have to fix SQLCli. + // TODO: fix SQLCli + @Ignore + @Test + public void testOmDB() throws Exception { + String dbOutPath = GenericTestUtils.getTempPath( + UUID.randomUUID() + "/out_sql.db"); + + String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS); + String dbPath = dbRootPath + "/" + OM_DB_NAME; + String[] args = {"-p", dbPath, "-o", dbOutPath}; + + cli.run(args); + + Connection conn = connectDB(dbOutPath); + String sql = "SELECT * FROM volumeList"; + ResultSet rs = executeQuery(conn, sql); + List<String> expectedValues = + new LinkedList<>(Arrays.asList(volumeName0, volumeName1)); + while (rs.next()) { + String userNameRs = rs.getString("userName"); + String volumeNameRs = rs.getString("volumeName"); + assertEquals(userName, userNameRs.substring(1)); + assertTrue(expectedValues.remove(volumeNameRs)); + } + assertEquals(0, expectedValues.size()); + + sql = "SELECT * FROM volumeInfo"; + rs = executeQuery(conn, sql); + expectedValues = + new LinkedList<>(Arrays.asList(volumeName0, volumeName1)); + while (rs.next()) { + String adName = rs.getString("adminName"); + String ownerName = rs.getString("ownerName"); + String volumeName = rs.getString("volumeName"); + assertEquals(adminName, adName); + assertEquals(userName, ownerName); + assertTrue(expectedValues.remove(volumeName)); + } + assertEquals(0, expectedValues.size()); + + sql = "SELECT * FROM aclInfo"; + rs = executeQuery(conn, sql); + expectedValues = + new LinkedList<>(Arrays.asList(volumeName0, volumeName1)); + while (rs.next()) { + String adName = rs.getString("adminName"); + String ownerName = rs.getString("ownerName"); + String volumeName = rs.getString("volumeName"); + String type = rs.getString("type"); + String uName = rs.getString("userName"); + String rights = rs.getString("rights"); + assertEquals(adminName, adName); + assertEquals(userName, ownerName); + assertEquals("USER", type); + assertEquals(userName, uName); + assertEquals("READ_WRITE", rights); + assertTrue(expectedValues.remove(volumeName)); + } + assertEquals(0, expectedValues.size()); + + sql = "SELECT * FROM bucketInfo"; + rs = executeQuery(conn, sql); + HashMap<String, String> expectedMap = new HashMap<>(); + expectedMap.put(bucketName0, volumeName0); + expectedMap.put(bucketName2, volumeName0); + expectedMap.put(bucketName1, volumeName1); + while (rs.next()) { + String volumeName = rs.getString("volumeName"); + String bucketName = rs.getString("bucketName"); + boolean versionEnabled = rs.getBoolean("versionEnabled"); + String storegeType = rs.getString("storageType"); + assertEquals(volumeName, expectedMap.remove(bucketName)); + assertFalse(versionEnabled); + assertEquals("DISK", storegeType); + } + assertEquals(0, expectedMap.size()); + + sql = "SELECT * FROM keyInfo"; + rs = executeQuery(conn, sql); + HashMap<String, List<String>> expectedMap2 = new HashMap<>(); + // no data written, data size will be 0 + expectedMap2.put(keyName0, + Arrays.asList(volumeName0, bucketName0, "0")); + expectedMap2.put(keyName1, + Arrays.asList(volumeName1, bucketName1, "0")); + expectedMap2.put(keyName2, + Arrays.asList(volumeName0, bucketName2, "0")); + expectedMap2.put(keyName3, + Arrays.asList(volumeName0, bucketName2, "0")); + while (rs.next()) { + String volumeName = rs.getString("volumeName"); + String bucketName = rs.getString("bucketName"); + String keyName = rs.getString("keyName"); + int dataSize = rs.getInt("dataSize"); + List<String> vals = expectedMap2.remove(keyName); + assertNotNull(vals); + assertEquals(vals.get(0), volumeName); + assertEquals(vals.get(1), bucketName); + assertEquals(vals.get(2), Integer.toString(dataSize)); + } + assertEquals(0, expectedMap2.size()); + + conn.close(); + Files.delete(Paths.get(dbOutPath)); + } + + private ResultSet executeQuery(Connection conn, String sql) + throws SQLException { + Statement stmt = conn.createStatement(); + return stmt.executeQuery(sql); + } + + private Connection connectDB(String dbPath) throws Exception { + Class.forName("org.sqlite.JDBC"); + String connectPath = + String.format("jdbc:sqlite:%s", dbPath); + return DriverManager.getConnection(connectPath); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java new file mode 100644 index 0000000..4026348 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright containerOwnership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.scm; + +import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.block.BlockManagerImpl; +import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy; +import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; +import org.apache.hadoop.ozone.scm.cli.SQLCLI; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.UUID; + +import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB; +import static org.apache.hadoop.ozone.OzoneConsts.KB; +import static org.junit.Assert.assertEquals; + +/** + * This class tests the CLI that transforms container into SQLite DB files. + */ +@RunWith(Parameterized.class) +public class TestContainerSQLCli { + + private EventQueue eventQueue; + + @Parameterized.Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, + {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB} + }); + } + + private static String metaStoreType; + + public TestContainerSQLCli(String type) { + metaStoreType = type; + } + + private static SQLCLI cli; + + private MiniOzoneCluster cluster; + private OzoneConfiguration conf; + private String datanodeIpAddress; + + private ContainerMapping mapping; + private NodeManager nodeManager; + private BlockManagerImpl blockManager; + + private HashMap<Long, Long> blockContainerMap; + + private final static long DEFAULT_BLOCK_SIZE = 4 * KB; + private static HddsProtos.ReplicationFactor factor; + private static HddsProtos.ReplicationType type; + private static final String CONTAINER_OWNER = "OZONE"; + + + @Before + public void setup() throws Exception { + blockContainerMap = new HashMap<>(); + + conf = new OzoneConfiguration(); + conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 2); + conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, + SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class); + if(conf.getBoolean(ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY, + ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT)){ + factor = HddsProtos.ReplicationFactor.THREE; + type = HddsProtos.ReplicationType.RATIS; + } else { + factor = HddsProtos.ReplicationFactor.ONE; + type = HddsProtos.ReplicationType.STAND_ALONE; + } + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(2).build(); + cluster.waitForClusterToBeReady(); + datanodeIpAddress = cluster.getHddsDatanodes().get(0) + .getDatanodeDetails().getIpAddress(); + cluster.getOzoneManager().stop(); + cluster.getStorageContainerManager().stop(); + eventQueue = new EventQueue(); + nodeManager = cluster.getStorageContainerManager().getScmNodeManager(); + mapping = new ContainerMapping(conf, nodeManager, 128, + eventQueue); + blockManager = new BlockManagerImpl(conf, nodeManager, mapping, eventQueue); + eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, blockManager); + eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, false); + GenericTestUtils.waitFor(() -> { + return !blockManager.isScmInChillMode(); + }, 10, 1000 * 15); + // blockManager.allocateBlock() will create containers if there is none + // stored in levelDB. The number of containers to create is the value of + // OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE which we set to 2. + // so the first allocateBlock() will create two containers. A random one + // is assigned for the block. + + // loop until both the two datanodes are up, try up to about 4 seconds. + for (int c = 0; c < 40; c++) { + if (nodeManager.getAllNodes().size() == 2) { + break; + } + Thread.sleep(100); + } + assertEquals(2, nodeManager.getAllNodes().size()); + AllocatedBlock ab1 = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, type, + factor, CONTAINER_OWNER); + blockContainerMap.put(ab1.getBlockID().getLocalID(), + ab1.getBlockID().getContainerID()); + + AllocatedBlock ab2; + // we want the two blocks on the two provisioned containers respectively, + // however blockManager picks containers randomly, keep retry until we + // assign the second block to the other container. This seems to be the only + // way to get the two containers. + // although each retry will create a block and assign to a container. So + // the size of blockContainerMap will vary each time the test is run. + while (true) { + ab2 = blockManager + .allocateBlock(DEFAULT_BLOCK_SIZE, type, factor, CONTAINER_OWNER); + blockContainerMap.put(ab2.getBlockID().getLocalID(), + ab2.getBlockID().getContainerID()); + if (ab1.getBlockID().getContainerID() != + ab2.getBlockID().getContainerID()) { + break; + } + } + + blockManager.close(); + mapping.close(); + nodeManager.close(); + + conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType); + cli = new SQLCLI(conf); + + } + + @After + public void shutdown() throws InterruptedException { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testConvertContainerDB() throws Exception { + String dbOutPath = GenericTestUtils.getTempPath( + UUID.randomUUID() + "/out_sql.db"); + // TODO : the following will fail due to empty Datanode list, need to fix. + //String dnUUID = cluster.getDataNodes().get(0).getUuid(); + String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS); + String dbPath = dbRootPath + "/" + SCM_CONTAINER_DB; + String[] args = {"-p", dbPath, "-o", dbOutPath}; + Connection conn; + String sql; + ResultSet rs; + + cli.run(args); + + //verify the sqlite db + // only checks the container names are as expected. Because other fields + // such as datanode UUID are generated randomly each time + conn = connectDB(dbOutPath); + sql = "SELECT * FROM containerInfo"; + rs = executeQuery(conn, sql); + ArrayList<Long> containerIDs = new ArrayList<>(); + while (rs.next()) { + containerIDs.add(rs.getLong("containerID")); + //assertEquals(dnUUID, rs.getString("leaderUUID")); + } + /* TODO: fix this later when the SQLCLI is fixed. + assertTrue(containerIDs.size() == 2 && + containerIDs.contains(pipeline1.getContainerName()) && + containerIDs.contains(pipeline2.getContainerName())); + + sql = "SELECT * FROM containerMembers"; + rs = executeQuery(conn, sql); + containerIDs = new ArrayList<>(); + while (rs.next()) { + containerIDs.add(rs.getLong("containerID")); + //assertEquals(dnUUID, rs.getString("datanodeUUID")); + } + assertTrue(containerIDs.size() == 2 && + containerIDs.contains(pipeline1.getContainerName()) && + containerIDs.contains(pipeline2.getContainerName())); + + sql = "SELECT * FROM datanodeInfo"; + rs = executeQuery(conn, sql); + int count = 0; + while (rs.next()) { + assertEquals(datanodeIpAddress, rs.getString("ipAddress")); + //assertEquals(dnUUID, rs.getString("datanodeUUID")); + count += 1; + } + // the two containers maybe on the same datanode, maybe not. + int expected = pipeline1.getLeader().getUuid().equals( + pipeline2.getLeader().getUuid())? 1 : 2; + assertEquals(expected, count); + */ + Files.delete(Paths.get(dbOutPath)); + } + + private ResultSet executeQuery(Connection conn, String sql) + throws SQLException { + Statement stmt = conn.createStatement(); + return stmt.executeQuery(sql); + } + + private Connection connectDB(String dbPath) throws Exception { + Class.forName("org.sqlite.JDBC"); + String connectPath = + String.format("jdbc:sqlite:%s", dbPath); + return DriverManager.getConnection(connectPath); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 794fdcc..8d08703 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -618,6 +618,11 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-datanode</artifactId> + <version>${hdds.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-ozone-integration-test</artifactId> <version>${hdds.version}</version> <type>test-jar</type> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
