http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_topology.py ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_topology.py b/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_topology.py deleted file mode 100755 index 0d90789..0000000 --- a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_topology.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -from resource_management import * -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import get_unique_id_and_date -from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ - FILE_TYPE_JAAS_CONF -from resource_management.core.shell import call -from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions.check_process_status import check_process_status -from resource_management.libraries.functions.format import format -from resource_management.libraries.functions.validate import call_and_match_output - -from actions import * - -class EagleTopology(Script): - # def get_stack_to_component(self): - # return {"HDP": "EAGLE-TOPOLOGY"} - - def install(self, env): - Logger.info('Install the eagle topology') - # self.install_packages(env) - import params - env.set_params(params) - self.configure(env) - # eagle_topology_exec(action = 'init') - - def configure(self,env): - Logger.info("Configure eagle topology") - import params - env.set_params(params) - - def pre_rolling_restart(self,env): - Logger.info("Executing Rolling Upgrade pre-restart") - import params - env.set_params(params) - - # if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - # Execute(format("hdp-select set eagle-topology {version}")) - - def stop(self, env): - Logger.info('Stop the eagle topology') - import params - env.set_params(params) - self.configure(env) - eagle_topology_exec(action = 'stop') - - def start(self, env): - Logger.info('Start the eagle topology') - import params - env.set_params(params) - self.configure(env) - eagle_topology_exec(action = 'start') - - def status(self, env): - Logger.info('Status of the eagle topology') - import params - env.set_params(params) - self.configure(env) - eagle_topology_exec(action = 'status') - -if __name__ == "__main__": - EagleTopology().execute()
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_topology_check.py ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_topology_check.py b/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_topology_check.py deleted file mode 100644 index ac28fad..0000000 --- a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_topology_check.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from resource_management import * - -class EagleTopologyCheck(Script): - def service_check(self,env): - import params - env.set_params(params) - import actions - eagle_service_exec(action="status") - -if __name__ == "__main__": - EagleTopologyCheck().execute() http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_userprofile_scheduler.py ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_userprofile_scheduler.py b/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_userprofile_scheduler.py deleted file mode 100755 index f2d5289..0000000 --- a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_userprofile_scheduler.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -from resource_management import * -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import get_unique_id_and_date -from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ - FILE_TYPE_JAAS_CONF -from resource_management.core.shell import call -from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions.check_process_status import check_process_status -from resource_management.libraries.functions.format import format -from resource_management.libraries.functions.validate import call_and_match_output - -from actions import * - -class EagleUserProfileScheduler(Script): - # def get_stack_to_component(self): - # return {"HDP": "EAGLE-TOPOLOGY"} - - def install(self, env): - Logger.info('Install Eagle UserProfile Scheduler') - # self.install_packages(env) - import params - env.set_params(params) - self.configure(env) - # eagle_topology_exec(action = 'init') - - def configure(self,env): - Logger.info("Configure Eagle UserProfile Scheduler") - import params - env.set_params(params) - - def pre_rolling_restart(self,env): - Logger.info("Executing rolling pre-restart Eagle UserProfile Scheduler") - import params - env.set_params(params) - - # if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - # Execute(format("hdp-select set eagle-topology {version}")) - - def stop(self, env): - Logger.info('Stopping Eagle UserProfile Scheduler') - import params - env.set_params(params) - self.configure(env) - eagle_userprofile_scheduler_exec(action = 'stop') - - def start(self, env): - Logger.info('Starting Eagle UserProfile Scheduler') - import params - env.set_params(params) - self.configure(env) - eagle_userprofile_scheduler_exec(action = 'start') - - def status(self, env): - Logger.info('Checking Eagle UserProfile Scheduler') - import params - env.set_params(params) - self.configure(env) - eagle_userprofile_scheduler_exec(action = 'status') - -if __name__ == "__main__": - EagleUserProfileScheduler().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_userprofile_topology.py ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_userprofile_topology.py b/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_userprofile_topology.py deleted file mode 100755 index 3671ff6..0000000 --- a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/eagle_userprofile_topology.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -from resource_management import * -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import get_unique_id_and_date -from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version -from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ - FILE_TYPE_JAAS_CONF -from resource_management.core.shell import call -from resource_management.core.logger import Logger -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions.check_process_status import check_process_status -from resource_management.libraries.functions.format import format -from resource_management.libraries.functions.validate import call_and_match_output - -from actions import * - -class EagleUserProfileTopology(Script): - # def get_stack_to_component(self): - # return {"HDP": "EAGLE-TOPOLOGY"} - - def install(self, env): - Logger.info('Install Eagle DAM UserProfile Topology') - # self.install_packages(env) - import params - env.set_params(params) - self.configure(env) - # eagle_hdfs_topology_exec(action = 'init') - - - def configure(self,env): - Logger.info("Configure Eagle DAM UserProfile Topology") - import params - env.set_params(params) - # eagle_hdfs_topology_exec(action = 'init') - - def pre_rolling_restart(self,env): - Logger.info("Executing rolling pre-restart Eagle DAM UserProfile Topology") - import params - env.set_params(params) - - # if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: - # Execute(format("hdp-select set eagle-topology {version}")) - - def stop(self, env): - Logger.info('Stopping Eagle DAM UserProfile Topology') - import params - env.set_params(params) - self.configure(env) - eagle_userprofile_topology_exec(action = 'stop') - - def start(self, env): - Logger.info('Starting Eagle DAM UserProfile Topology') - import params - env.set_params(params) - self.configure(env) - - eagle_userprofile_topology_exec(action = 'init') - eagle_userprofile_topology_exec(action = 'start') - - def status(self, env): - Logger.info('Checking Eagle DAM UserProfile Topology status') - import params - env.set_params(params) - self.configure(env) - eagle_userprofile_topology_exec(action = 'status') - -if __name__ == "__main__": - EagleUserProfileTopology().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/params.py b/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/params.py deleted file mode 100755 index 1e0f839..0000000 --- a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/params.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Ambari Agent -""" - -from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions -from resource_management.libraries.functions.default import default -from resource_management import * -# import status_params - -# server configurations -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -stack_version_unformatted = str(config['hostLevelParams']['stack_version']) -hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) - -stack_name = default("/hostLevelParams/stack_name", None) - -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade -version = default("/commandParams/version", None) - -#hadoop params -if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: - role_root = "eagle" - command_role = default("/role", "") - - if command_role == "EAGLE_SERVICE": - role_root = "eagle" - - eagle_home = format("/usr/hdp/current/{role_root}") - eagle_bin = format("/usr/hdp/current/{role_root}/bin") - eagle_conf = format("/usr/hdp/current/{role_root}/conf") -else: - eagle_home = "/usr" - eagle_bin = "/usr/lib/zookeeper/bin" - -eagle_user = 'root' -eagle_site = 'sandbox' -eagle_service_pid_file = format("{eagle_home}/temp/service.pid") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/service_check.py b/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/service_check.py deleted file mode 100644 index b84ffc0..0000000 --- a/eagle-external/eagle-ambari/lib/EAGLE/package/scripts/service_check.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from resource_management import * -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.validate import call_and_match_output -from resource_management.libraries.functions.format import format -from resource_management.core.logger import Logger - -class EagleServiceCheck(Script): - def service_check(self,env): - Logger.info("Checking eagle service") - import params - env.set_params(params) - check_eagle_service_cmd=format("ls {eagle_service_pid_file} >/dev/null 2>&1 && ps -p `cat {eagle_service_pid_file}` >/dev/null 2>&1") - Execute(check_eagle_service_cmd,logoutput=True,try_sleep=3, tries=5) - -if __name__ == "__main__": - EagleServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-ambari/lib/metainfo.xml ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-ambari/lib/metainfo.xml b/eagle-external/eagle-ambari/lib/metainfo.xml deleted file mode 100644 index 0953c8f..0000000 --- a/eagle-external/eagle-ambari/lib/metainfo.xml +++ /dev/null @@ -1,27 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>EAGLE</name> - <extends>common-services/EAGLE/0.0.1</extends> - </service> - </services> -</metainfo> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/Dockerfile ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/Dockerfile b/eagle-external/eagle-docker/Dockerfile deleted file mode 100644 index 4aa3dec..0000000 --- a/eagle-external/eagle-docker/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTICE: This docker image is built based sequenceiq/ambari which is open sourced under github: https://github.com/sequenceiq/docker-ambari -FROM sequenceiq/ambari:1.7.0 - -MAINTAINER d...@eagle.incubator.apache.org - -# Eagle environment -ENV EAGLE_HOME=/usr/hdp/current/eagle - -# Install Eagle Package -# -# [Option #1] Download eagle package from external link -# ENV EAGLE_DOWNLOAD_LINK http://66.211.190.194/eagle-0.1.0.tar.gz -# RUN curl -sL $EAGLE_DOWNLOAD_LINK | tar -xz -C /usr/hdp/current -# RUN cd /usr/hdp/current && ln -s ./eagle-0.1.0 eagle - -# [Option #2] Load eagle package from locally built, -# In such way, should build with `eagle-docker.sh buid` instead of executing native -# `docker build` directly, to prepare the eagle package into build/eagle -ADD target/eagle-current /usr/hdp/current/eagle - -# Load External Packages -RUN yum install -y kafka zookeeper storm hbase tez hadoop snappy snappy-devel hadoop-libhdfs ambari-log4j hive hive-hcatalog hive-webhcat webhcat-tar-hive webhcat-tar-pig mysql-connector-java mysql-server - -# Load Static Resources -ADD resource/serf /usr/local/serf -ADD resource/install-cluster.sh /tmp/ -ADD resource/eagle-singlenode.json /tmp/ -ADD resource/eagle-multinode.json /tmp/ -ADD resource/wait-for-eagle.sh /tmp/ -ADD resource/deploy-eagle.sh /usr/hdp/current/eagle/deploy.sh - -EXPOSE 9099 8744 8080 2181 2888 6667 60020 60030 60010 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/LICENSE ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/LICENSE b/eagle-external/eagle-docker/LICENSE deleted file mode 100644 index b9c8eef..0000000 --- a/eagle-external/eagle-docker/LICENSE +++ /dev/null @@ -1,209 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -============================================================================== -Apache Eagle (incubating) Subcomponents: - -The Apache Eagle project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/README.md ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/README.md b/eagle-external/eagle-docker/README.md deleted file mode 100644 index 105fee0..0000000 --- a/eagle-external/eagle-docker/README.md +++ /dev/null @@ -1,125 +0,0 @@ -<!-- -{% comment %} -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to you under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -{% endcomment %} ---> - -# Eagle in Docker - -> Docker image for Apache Eagle https://hub.docker.com/r/apacheeagle/sandbox - -This is docker container for eagle to help users to have a quick preview about eagle features. -And this project is to build apache/eagle images and provide eagle-functions to start the containers of eagle. - -## Prerequisite -* [Apache Maven](https://maven.apache.org) -* [NPM](https://www.npmjs.com) -* [Docker](https://www.docker.com) - * [Mac OS X](http://docs.docker.com/mac/started) - * [Linux](http://docs.docker.com/linux/started) - * [Windows](http://docs.docker.com/windows/started) - -## Getting Started -The fastest way to get started with Eagle is to run with [docker](https://github.com/docker/docker) by one of following options: - -* Pull latest eagle docker image from [docker hub](https://hub.docker.com/r/apacheeagle/sandbox/) directly: - - docker pull apacheeagle/sandbox - - Then run eagle docker image: - - docker run -p 9099:9099 -p 8080:8080 -p 8744:8744 -p 2181:2181 -p 2888:2888 -p 6667:6667 -p 60020:60020 \ - -p 60030:60030 -p 60010:60010 -d --dns 127.0.0.1 --entrypoint /usr/local/serf/bin/start-serf-agent.sh \ - -e KEYCHAIN= --env EAGLE_SERVER_HOST=sandbox.eagle.incubator.apache.org --name sandbox \ - -h sandbox.eagle.incubator.apache.org --privileged=true apacheeagle/sandbox:latest \ - --tag ambari-server=true - docker run -it --rm -e EXPECTED_HOST_COUNT=1 -e BLUEPRINT=hdp-singlenode-eagle --link sandbox:ambariserver\ - --entrypoint /bin/sh apacheeagle/sandbox:latest -c /tmp/install-cluster.sh - - -* Build eagle docker image from source code with [eagle-docker](eagle-external/eagle-docker): - - git clone https://github.com/apache/incubator-eagle.git - cd incubator-eagle && ./eagle-docker boot - -## Usage ## -Basic usage of the entry script of eagle-docker: [bin/eagle-docker.sh](bin/eagle-docker.sh) - - Usage: ./eagle-docker [options] [command] - - Apache Eagle Docker Image : apacheeagle/sandbox:latest - - Commands: - build Build eagle docker image - deploy Deploy eagle docker image - start Start eagle docker instance - stop Stop eagle docker instance - status List eagle docker image and instance status - clean Clean docker image and instance - shell Execute docker instance bash, default: eagle-sandbox - boot Simply bootstrap eagle docker by building then deploying - - Options: - --node [num] Docker instances node number, default is 1 - --help Display eagle docker image usage information - -## Advanced -1. **Build Image**: Go to the root directory where the [Dockerfile](Dockerfile) is in, build image with following command: - - docker built -t apacheeagle/sandbox . - - > The docker image is named `apacheeagle/sandbox`. Eagle docker image is based on [`ambari:1.7.0`](https://github.com/sequenceiq/docker-ambari), it will install ganglia, hbase,hive,storm,kafka and so on in this image. Add startup script and buleprint file into image. - -2. **Verify Image**: After building the `apacheeagle/sandbox` image successfully, verify the images and could find eagle image. - - docker images - -3. **Deploy Image**: This project also provides helper functions in script [eagle-lib.sh](bin/eagle-lib.sh) for convenience. - - # Firstly, load the helper functions into context - source eagle-functions - - # Secondly, start to deploy eagle cluster - - # (1) start single-node container - eagle-deploy-cluster 1 - - # (2) Or muti-node containers - eagle-deploy-cluster 3 - -4. **Find IP and Port Mapping**: After the container is up and running. The first thing you need to do is finding the IP address and port mapping of the docker container: - - docker inspect -f '{{ .NetworkSettings.IPAddress }}' eagle-server - docker ps - -5. **Start to use Eagle**: Congratulations! You are able to start using Eagle now. Please open eagle ui at following address (username: ADMIN, password: secret by default) - - http://{{container_ip}}:9099/eagle-service - -6. **Manage Eagle Cluster**: This step is about how to managing the eagle cluster though not must-have at starting. Eagle docker depends on Ambari to manage the cluster infrastructure of Eagle. Following are some helpful links: - - * Ambari UI: `http://{{container_ip}}:8080` (username: ADMIN, password: ADMIN) - * Storm UI: `http://{{container_ip}}:8744` - -## Get Help -The fastest way to get response from eagle community is to send email to the mail list [d...@eagle.incubator.apache.org](mailto:d...@eagle.incubator.apache.org), -and remember to subscribe our mail list via [dev-subscr...@eagle.incubator.apache.org](mailto:dev-subscr...@eagle.incubator.apache.org) - -## FAQ -[https://cwiki.apache.org/confluence/display/EAG/FAQ#FAQ-EagleDocker](https://cwiki.apache.org/confluence/display/EAG/FAQ#FAQ-EagleDocker) - -## License -Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). More details, please refer to [LICENSE](LICENSE) file. http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/bin/eagle-docker.sh ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/bin/eagle-docker.sh b/eagle-external/eagle-docker/bin/eagle-docker.sh deleted file mode 100755 index d490d21..0000000 --- a/eagle-external/eagle-docker/bin/eagle-docker.sh +++ /dev/null @@ -1,299 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTICE: This script is developed and maintained by Apache Eagle community under Apache Softwarw Foundation but not from official Docker product or community. - -EAGLE_DOCKER_VERSION=latest -EAGLE_DOCKER_NAME=apacheeagle/sandbox -EAGLE_DOCKER_PREFIX=sandbox - -export NODE_NUM=1 - -cd `dirname $0`/../ -source bin/eagle-lib.sh - -function check_env(){ - which docker 1>/dev/null 2>&1 - - if [ $? != 0 ];then - echo "[ERROR] docker commnad is not found, please make sure install and configure Docker Engine properly, docker docs: http://docs.docker.com"; 1>&2 - exit 1 - else - log=`docker images 2>&1` - if [ $? != 0 ];then - echo "[ERROR] $log" - exit 1 - fi - fi -} - -function usage() { - echo "Usage: ./eagle-docker [options] [command]" - - echo "" - echo "Apache Eagle Docker Image : ${EAGLE_DOCKER_NAME}:${EAGLE_DOCKER_VERSION}" - echo "" - echo "Commands:" - echo " build Build eagle docker image" - echo " deploy Deploy eagle docker image" - echo " start Start eagle docker instance" - echo " stop Stop eagle docker instance" - echo " status List eagle docker image and instance status" - echo " clean Clean docker image and instance" - echo " shell Execute docker instance bash, default: eagle-sandbox" - echo " boot Simply bootstrap eagle docker by building then deploying" - echo "" - echo "Options:" - echo " --node [num] Docker instances node number, default is 1" - echo " --help Display eagle docker image usage information" - echo "" - exit 0 -} - -function build(){ - check_env - # ========================================== - # Check Eagle Docker Image - # ========================================== - echo "[1/3] Validate Environment" - # echo "Checking eagle docker image ..." - docker images | grep $EAGLE_DOCKER_NAME 1>/dev/null 2>&1 - if [ $? == 0 ];then - echo "Eagle docker image already exists:" - echo "" - docker images | grep $EAGLE_DOCKER_NAME - echo "" - echo "Rebuild it now [Y/N]? " - read rebuild - if [ $rebuild == "Y" ] || [ $rebuild == "y" ];then - echo "Clean eagle image" - clean - else - echo "Quit now" - exit 1 - fi - else - echo "No existing eagle docker images, environment is ready" - fi - - # ========================================== - # Build Eagle Binary Package - # ========================================== - echo "[2/3] Build Eagle Binary Package" - # echo "Checking eagle binary package" - cd ../../ - ls eagle-assembly/target/eagle-*-bin.tar.gz 1>/dev/null 2>&1 - if [ $? == 0 ];then - eagle_bin_pkg=`ls eagle-assembly/target/eagle-*-bin.tar.gz` - echo "Found eagle binary package at $eagle_bin_pkg" - else - echo "Eagle binary package is not found" - echo "Build eagle binary package now" - # ========================================== - # Build Eagle Binary Package with Maven - # ========================================== - echo "" - echo "Execute: mvn package -DskipTests " - mvn package -DskipTests - if [ $? == 0 ];then - echo "Built successfully existing with 0" - ls eagle-assembly/target/eagle-*-bin.tar.gz 1>/dev/null 2>&1 - if [ $? == 0 ];then - eagle_bin_pkg=`ls eagle-assembly/target/eagle-*-bin.tar.gz` - echo "" - echo "[SUCCESS] Successfully build eagle binary package at $eagle_bin_pkg" - else - echo "" - echo "[FAILED] Built eagle binary package exiting with 0, but package is not found" - exit 1 - fi - else - echo "" - echo "[FAILED] Failed to build eagle binary package, exit 1" - exit 1 - fi - fi - - # ===================================== - # Build Eagle Docker Image - # ===================================== - echo "[3/3] Build Eagle Docker Image: $EAGLE_DOCKER_NAME" - echo "Extracting $eagle_bin_pkg" - if [ -e eagle-external/eagle-docker/target ];then - rm -rf eagle-external/eagle-docker/target - fi - mkdir -p eagle-external/eagle-docker/target/eagle-current - - out=`tar -xzf $eagle_bin_pkg -C eagle-external/eagle-docker/target/` - if [ $? != 0 ];then - echo "[ERROR] Failed to execute 'tar -xzf $eagle_bin_pkg -C eagle-external/eagle-docker/target/': $out" 1>&2 - exit 1 - fi - mv eagle-external/eagle-docker/target/eagle-*/* eagle-external/eagle-docker/target/eagle-current - - echo "Execute: docker build -t $EAGLE_DOCKER_NAME ." - cd eagle-external/eagle-docker - docker build -t $EAGLE_DOCKER_NAME . - - if [ $? == 0 ];then - echo "" - echo "[SUCCESS] Successfully built eagle docker image : $EAGLE_DOCKER_NAME" - else - echo "" - echo "[FAILED] Failed to build eagle docker image : $EAGLE_DOCKER_NAME" - fi -} - - -function start(){ - check_env - docker ps -a | grep $EAGLE_DOCKER_NAME 1>/dev/null - if [ $? != 0 ];then - echo "No eagle docker instance found" - exit 1 - fi - docker ps -a | grep $EAGLE_DOCKER_NAME | grep Exited 1>/dev/null - if [ $? != 0 ];then - echo "No exited eagle docker instance found, all should be running" - exit 1 - fi - docker ps -a | grep $EAGLE_DOCKER_NAME | grep Exited | awk '{ print $1 }' | while read id; do echo "Starting $id";docker start $id 1>/dev/null;done - - echo "" - echo "[Eagle Docker Instances (Up)]" - echo "" - docker ps | grep $EAGLE_DOCKER_NAME -} - -function stop(){ - check_env - docker ps | grep $EAGLE_DOCKER_NAME 1>/dev/null - if [ $? != 0 ];then - echo "No eagle docker instance is running" - exit 1 - fi - docker ps | grep $EAGLE_DOCKER_NAME | awk '{ print $1 }' | while read id; do echo "Stopping $id";docker kill $id;done - - echo "" - echo "[Eagle Docker Instances (Exited)]" - echo "" - docker ps -a| grep $EAGLE_DOCKER_NAME -} - -function status(){ - check_env - echo "[Eagle Docker Images]" - echo "" - docker images | grep $EAGLE_DOCKER_NAME - [[ $? == 0 ]] || echo "no images found for eagle" - echo "" - - echo "[Eagle Docker Instances]" - echo "" - docker ps -a | grep $EAGLE_DOCKER_NAME - [[ $? == 0 ]] || echo "no instances found eagle" - echo "" -} - - -function clean(){ - check_env - echo "[1/4] Stop eagle instances" - # Call in multi-process to avoid exiting by stopping - echo `stop` - - echo "[2/4] Remove eagle instances" - docker ps -a | grep $EAGLE_DOCKER_NAME |awk '{print $1}'| while read id; do echo "Removing instance $id";docker rm $id;done - - echo "[3/4] Remove eagle images" - docker images | grep $EAGLE_DOCKER_NAME | awk '{print $3}' | while read id; do echo "Removing image $id"; docker rmi $id;done - - echo "[4/4] Clean temp files" - [[ -e target/ ]] && rm -rf target/ - - echo "Cleaning Done." -} - -function deploy(){ - check_env - if [ "$NODE_NUM" == "" ];then - export NODE_NUM=1 - fi - echo "Deploying $NODE_NUM nodes" - eagle-deploy-cluster $NODE_NUM -} - -function boot(){ - check_env - build - deploy - status -} - -function exec_bash(){ - docker exec -it $EAGLE_DOCKER_PREFIX bash -} - -while [[ -n $1 ]]; do - case $1 in - "--node") - if [ $# -lt 2 ]; then - usage - exit 1 - fi - NODE_NUM=$2 - shift - ;; - "deploy") - deploy - exit - ;; - "build") - build - exit - ;; - "boot") - boot - exit - ;; - "status") - status - exit - ;; - "start") - start - exit - ;; - "stop") - stop - exit - ;; - "clean") - clean - exit - ;; - "shell") - exec_bash - exit - ;; - *) - usage - exit 1 - ;; - esac - shift -done http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/bin/eagle-lib.sh ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/bin/eagle-lib.sh b/eagle-external/eagle-docker/bin/eagle-lib.sh deleted file mode 100755 index c988866..0000000 --- a/eagle-external/eagle-docker/bin/eagle-lib.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -: ${VERSION:=latest} -: ${IMAGE:="apacheeagle/sandbox:${VERSION}"} - -: ${NODE_PREFIX:=sandbox} -: ${AMBARI_SERVER_NAME:=${NODE_PREFIX}} -: ${MYDOMAIN:=eagle.incubator.apache.org} -: ${DOCKER_OPTS:="--dns 127.0.0.1 --entrypoint /usr/local/serf/bin/start-serf-agent.sh -e KEYCHAIN=$KEYCHAIN --env EAGLE_SERVER_HOST=${AMBARI_SERVER_NAME}.${MYDOMAIN}"} -: ${CLUSTER_SIZE:=1} -: ${DEBUG:=1} -: ${SLEEP_TIME:=2} -: ${DRY_RUN:=false} - -run-command() { - CMD="$@" - if [ "$DRY_RUN" == "false" ]; then - debug "$CMD" - "$@" - else - debug [DRY_RUN] "$CMD" - fi -} - -amb-clean() { - unset NODE_PREFIX AMBARI_SERVER_NAME MYDOMAIN IMAGE DOCKER_OPTS DEBUG SLEEP_TIME AMBARI_SERVER_IP DRY_RUN -} - -get-ambari-server-ip() { - AMBARI_SERVER_IP=$(get-host-ip ${AMBARI_SERVER_NAME}) -} - -get-host-ip() { - HOST=$1 - docker inspect --format="{{.NetworkSettings.IPAddress}}" ${HOST} -} - -amb-members() { - get-ambari-server-ip - serf members --rpc-addr $(docker inspect --format "{{.NetworkSettings.IPAddress}}" ${AMBARI_SERVER_NAME}):7373 -} - -amb-settings() { - cat <<EOF - NODE_PREFIX=$NODE_PREFIX - MYDOMAIN=$MYDOMAIN - CLUSTER_SIZE=$CLUSTER_SIZE - AMBARI_SERVER_NAME=$AMBARI_SERVER_NAME - IMAGE=$IMAGE - DOCKER_OPTS=$DOCKER_OPTS - AMBARI_SERVER_IP=$AMBARI_SERVER_IP - DRY_RUN=$DRY_RUN -EOF -} - -debug() { - [ $DEBUG -gt 0 ] && echo [DEBUG] "$@" 1>&2 -} - -docker-ps() { - #docker ps|sed "s/ \{3,\}/#/g"|cut -d '#' -f 1,2,7|sed "s/#/\t/g" - docker inspect --format="{{.Name}} {{.NetworkSettings.IPAddress}} {{.Config.Image}} {{.Config.Entrypoint}} {{.Config.Cmd}}" $(docker ps -q) -} - -docker-psa() { - #docker ps|sed "s/ \{3,\}/#/g"|cut -d '#' -f 1,2,7|sed "s/#/\t/g" - docker inspect --format="{{.Name}} {{.NetworkSettings.IPAddress}} {{.Config.Image}} {{.Config.Entrypoint}} {{.Config.Cmd}}" $(docker ps -qa) -} - -amb-start-cluster() { - local act_cluster_size=$1 - : ${act_cluster_size:=$CLUSTER_SIZE} - echo starting an ambari cluster with: $act_cluster_size nodes - - amb-start-first - [ $act_cluster_size -gt 1 ] && for i in $(seq $((act_cluster_size - 1))); do - amb-start-node $i - done -} - -_amb_run_shell() { - COMMAND=$1 - : ${COMMAND:? required} - get-ambari-server-ip - NODES=$(docker inspect --format="{{.Config.Image}} {{.Name}}" $(docker ps -q)|grep $IMAGE|grep $NODE_PREFIX|wc -l|xargs) - run-command docker run -it --rm -e EXPECTED_HOST_COUNT=$NODES -e BLUEPRINT=$BLUEPRINT --link ${AMBARI_SERVER_NAME}:ambariserver --entrypoint /bin/sh $IMAGE -c $COMMAND -} - -amb-shell() { - _amb_run_shell /tmp/ambari-shell.sh -} - -eagle-deploy-cluster() { - local act_cluster_size=$1 - : ${act_cluster_size:=$CLUSTER_SIZE} - - if [ $# -gt 1 ]; then - BLUEPRINT=$2 - else - [ $act_cluster_size -gt 1 ] && BLUEPRINT=hdp-multinode-eagle || BLUEPRINT=hdp-singlenode-eagle - fi - - : ${BLUEPRINT:?" required (hdp-singlenode-eagle / hdp-multinode-eagle)"} - - amb-start-cluster $act_cluster_size - _amb_run_shell /tmp/install-cluster.sh -} - -amb-start-first() { - run-command docker run -p 9099:9099 -p 8080:8080 -p 8744:8744 -p 2181:2181 -p 2888:2888 -p 6667:6667 -p 60020:60020 -p 60030:60030 -p 60010:60010 -d $DOCKER_OPTS --name $AMBARI_SERVER_NAME -h $AMBARI_SERVER_NAME.$MYDOMAIN --privileged=true $IMAGE --tag ambari-server=true -} - -amb-copy-to-hdfs() { - get-ambari-server-ip - FILE_PATH=${1:?"usage: <FILE_PATH> <NEW_FILE_NAME_ON_HDFS> <HDFS_PATH>"} - FILE_NAME=${2:?"usage: <FILE_PATH> <NEW_FILE_NAME_ON_HDFS> <HDFS_PATH>"} - DIR=${3:?"usage: <FILE_PATH> <NEW_FILE_NAME_ON_HDFS> <HDFS_PATH>"} - amb-create-hdfs-dir $DIR - DATANODE=$(curl -si -X PUT "http://$AMBARI_SERVER_IP:50070/webhdfs/v1$DIR/$FILE_NAME?user.name=hdfs&op=CREATE"; |grep Location | sed "s/\..*//; s@.*http://@@";) - DATANODE_IP=$(get-host-ip $DATANODE) - curl -T $FILE_PATH "http://$DATANODE_IP:50075/webhdfs/v1$DIR/$FILE_NAME?op=CREATE&user.name=hdfs&overwrite=true&namenoderpcaddress=$AMBARI_SERVER_IP:8020"; -} - -amb-create-hdfs-dir() { - get-ambari-server-ip - DIR=$1 - curl -X PUT "http://$AMBARI_SERVER_IP:50070/webhdfs/v1$DIR?user.name=hdfs&op=MKDIRS"; > /dev/null 2>&1 -} - -amb-scp-to-first() { - get-ambari-server-ip - FILE_PATH=${1:?"usage: <FILE_PATH> <DESTINATION_PATH>"} - DEST_PATH=${2:?"usage: <FILE_PATH> <DESTINATION_PATH>"} - scp $FILE_PATH root@$AMBARI_SERVER_IP:$DEST_PATH -} - -amb-start-node() { - get-ambari-server-ip - : ${AMBARI_SERVER_IP:?"AMBARI_SERVER_IP is needed"} - NUMBER=${1:?"please give a <NUMBER> parameter it will be used as node<NUMBER>"} - if [ $# -eq 1 ] ;then - MORE_OPTIONS="-d" - else - shift - MORE_OPTIONS="$@" - fi - run-command docker run $MORE_OPTIONS -e SERF_JOIN_IP=$AMBARI_SERVER_IP $DOCKER_OPTS --name ${NODE_PREFIX}_$NUMBER -h ${NODE_PREFIX}_${NUMBER}.$MYDOMAIN $IMAGE --log-level debug -} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/deploy-eagle.sh ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/resource/deploy-eagle.sh b/eagle-external/eagle-docker/resource/deploy-eagle.sh deleted file mode 100755 index a1e5a19..0000000 --- a/eagle-external/eagle-docker/resource/deploy-eagle.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o pipefail # trace ERR through pipes -set -o errtrace # trace ERR through 'time command' and other functions - -function error() { -SCRIPT="$0" # script name -LASTLINE="$1" # line of error occurrence -LASTERR="$2" # error code -echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code ${LASTERR}" -exit 1 -} - -trap 'error ${LINENO} ${?}' ERR - -echo "" -echo "Welcome to try Eagle" -echo "" - -echo "Eagle home folder path is $EAGLE_HOME" -cd $EAGLE_HOME - - -#Initializing Eagle Service ... -sh ./bin/eagle-service-init.sh - -sleep 10 - -#Starting Eagle Service ... -sh ./bin/eagle-service.sh start - -sleep 10 - -echo "Creating kafka topics for eagle ... " -KAFKA_HOME=/usr/hdp/current/kafka-broker -EAGLE_ZOOKEEPER_QUORUM=$EAGLE_SERVER_HOST:2181 -topic=`${KAFKA_HOME}/bin/kafka-topics.sh --list --zookeeper $EAGLE_ZOOKEEPER_QUORUM --topic sandbox_hdfs_audit_log` -if [ -z $topic ]; then - $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $EAGLE_ZOOKEEPER_QUORUM --replication-factor 1 --partitions 1 --topic sandbox_hdfs_audit_log -fi - -if [ $? = 0 ]; then -echo "==> Create kafka topic successfully for eagle" -else -echo "==> Failed, exiting" -exit 1 -fi - -EAGLE_NIMBUS_HOST=$EAGLE_SERVER_HOST -EAGLE_SERVICE_HOST=$EAGLE_SERVER_HOST -EAGLE_TOPOLOGY_JAR=`ls ${EAGLE_HOME}/lib/topology/eagle-topology-*-assembly.jar` - -${EAGLE_HOME}/bin/eagle-topology-init.sh -[ $? != 0 ] && exit 1 -${EAGLE_HOME}/examples/sample-sensitivity-resource-create.sh -[ $? != 0 ] && exit 1 -${EAGLE_HOME}/examples/sample-policy-create.sh -[ $? != 0 ] && exit 1 -storm jar $EAGLE_TOPOLOGY_JAR org.apache.eagle.security.auditlog.HdfsAuditLogProcessorMain -D config.file=${EAGLE_HOME}/conf/sandbox-hdfsAuditLog-application.conf -D eagleProps.eagleService.host=$EAGLE_SERVICE_HOST -[ $? != 0 ] && exit 1 -storm jar $EAGLE_TOPOLOGY_JAR org.apache.eagle.security.hive.jobrunning.HiveJobRunningMonitoringMain -D config.file=${EAGLE_HOME}/conf/sandbox-hiveQueryLog-application.conf -D eagleProps.eagleService.host=$EAGLE_SERVICE_HOST -[ $? != 0 ] && exit 1 -storm jar $EAGLE_TOPOLOGY_JAR org.apache.eagle.security.userprofile.UserProfileDetectionMain -D config.file=${EAGLE_HOME}/conf/sandbox-userprofile-topology.conf -D eagleProps.eagleService.host=$EAGLE_SERVICE_HOST -[ $? != 0 ] && exit 1 - -# TODO: More eagle start - -echo "Eagle is deployed successfully!" - -echo "Please visit http://<container_ip>:9099/eagle-service to play with Eagle!" http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/eagle-multinode.json ---------------------------------------------------------------------- diff --git a/eagle-external/eagle-docker/resource/eagle-multinode.json b/eagle-external/eagle-docker/resource/eagle-multinode.json deleted file mode 100644 index 6d6da67..0000000 --- a/eagle-external/eagle-docker/resource/eagle-multinode.json +++ /dev/null @@ -1,163 +0,0 @@ -{ -"configurations": [ - { - "hdfs-site": { - "dfs.permissions.enabled": "false" - }, - "hive-site": { - "javax.jdo.option.ConnectionUserName": "hive", - "javax.jdo.option.ConnectionPassword": "hive" - } - }, - { - "hadoop-env": { - "properties" : { - "content" : "\r\n# Set Hadoop-specific environment variables here.\r\n\r\n# The only required environment variable is JAVA_HOME. All others are\r\n# optional. When running a distributed configuration it is best to\r\n# set JAVA_HOME in this file, so that it is correctly defined on\r\n# remote nodes.\r\n\r\n# The java implementation to use. Required.\r\nexport JAVA_HOME={{java_home}}\r\nexport HADOOP_HOME_WARN_SUPPRESS=1\r\n\r\n# Hadoop home directory\r\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\r\n\r\n# Hadoop Configuration Directory\r\n\r\n{# this is different for HDP1 #}\r\n# Path to jsvc required by secure HDP 2.0 datanode\r\nexport JSVC_HOME={{jsvc_path}}\r\n\r\n\r\n# The maximum amount of heap to use, in MB. Default is 1000.\r\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\r\n\r\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\r\n\r\n# Extra Java runtime options. Empty by default.\r\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=tr ue ${HADOOP_OPTS}\"\r\n\r\n# Command specific options appended to HADOOP_OPTS when specified\r\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT,KAFKA_HDFS_AUDIT ${HADOOP_NAMENODE_OPTS}\"\r\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d %H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\r\n\r\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\r\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\r\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCE R_OPTS}\"\r\n\r\nexport HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\r\n\r\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\r\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\r\n\r\n# On secure datanodes, user to run the datanode as after dropping privileges\r\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\r\n\r\n# Extra ssh options. Empty by default.\r\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\r\n\r\n# Where log files are stored. $HADOOP_HOME/logs by default.\r\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\r\n\r\n# History server logs\r\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\r\n\r\n# Where log files are stored in the secure data environment.\r\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\r\n\r\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\r\n# e xport HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\r\n\r\n# host:path where hadoop code should be rsync'd from. Unset by default.\r\n# export HADOOP_MASTER=master:/home/$USER/src/hadoop\r\n\r\n# Seconds to sleep between slave commands. Unset by default. This\r\n# can be useful in large clusters, where, e.g., slave rsyncs can\r\n# otherwise arrive faster than the master can service them.\r\n# export HADOOP_SLAVE_SLEEP=0.1\r\n\r\n# The directory where pid files are stored. /tmp by default.\r\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\r\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\r\n\r\n# History server pid\r\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\r\n\r\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\r\n\r\n# A string representing this instance of hadoop. $USER by default.\r\nexport HADOOP_IDENT_STRING=$USER\r\n\r\n# The scheduling priority for daemon processes. See 'm an nice'.\r\n\r\n# export HADOOP_NICENESS=10\r\n\r\n# Use libraries from standard classpath\r\nJAVA_JDBC_LIBS=\"\"\r\n#Add libraries required by mysql connector\r\nfor jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`\r\ndo\r\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\r\ndone\r\n# Add libraries required by oracle connector\r\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\r\ndo\r\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\r\ndone\r\n# Add libraries required by nodemanager\r\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\r\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}:/usr/hdp/current/eagle/lib/log4jkafka/lib/*\r\n\r\n# added to the HADOOP_CLASSPATH\r\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\r\n if [ -d \"/etc/tez/conf/\" ]; then\r\n # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.\r\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez -client/lib/*:/etc/tez/conf/\r\n fi\r\nfi\r\n\r\n# Setting path to hdfs command line\r\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\r\n\r\n# Mostly required for hadoop 2.0\r\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\r\n\r\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\r\n" - } - } - }, - { - "hdfs-log4j": { - "properties": { - "content" : "\r\n#\r\n# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements. See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership. The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License. You may obtain a copy of the License at\r\n#\r\n# http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied. See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n#\r\n\r\n\r\n# Define some default values that can be overridden by system properties\r\n# To change daemon root logger use hadoo p_root_logger in hadoop-env\r\nhadoop.root.logger=INFO,console\r\nhadoop.log.dir=.\r\nhadoop.log.file=hadoop.log\r\n\r\n\r\n# Define the root logger to the system property \"hadoop.root.logger\".\r\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\r\n\r\n# Logging Threshold\r\nlog4j.threshhold=ALL\r\n\r\n#\r\n# Daily Rolling File Appender\r\n#\r\n\r\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Rollver at midnight\r\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\r\n\r\n# 30-day backup\r\n#log4j.appender.DRFA.MaxBackupIndex=30\r\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\r\n\r\n# Pattern format: Date LogLevel LoggerName LogMessage\r\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n# Debugging Pattern format\r\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n#\r\n# console\r\n# Add \"console\" to rootlog ger above if you want to use this\r\n#\r\n\r\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\r\nlog4j.appender.console.target=System.err\r\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\r\n\r\n#\r\n# TaskLog Appender\r\n#\r\n\r\n#Default values\r\nhadoop.tasklog.taskid=null\r\nhadoop.tasklog.iscleanup=false\r\nhadoop.tasklog.noKeepSplits=4\r\nhadoop.tasklog.totalLogFileSize=100\r\nhadoop.tasklog.purgeLogSplits=true\r\nhadoop.tasklog.logsRetainHours=12\r\n\r\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\r\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\r\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\r\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\r\n\r\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n\r\n#\r\n#Security audit appender\r\ n#\r\nhadoop.security.logger=INFO,console\r\nhadoop.security.log.maxfilesize=256MB\r\nhadoop.security.log.maxbackupindex=20\r\nlog4j.category.SecurityLogger=${hadoop.security.logger}\r\nhadoop.security.log.file=SecurityAuth.audit\r\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\r\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\r\n\r\n#\r\n# hdfs audit l ogging\r\n#\r\nhdfs.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\r\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\r\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.eagle.incubator.apache.org:6667\r\nlog4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.KAFKA_HDFS_AUDIT .Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1\r\nlog4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1\r\n\r\n#\r\n# mapred audit logging\r\n#\r\nmapred.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\r\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\r\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\n#\r\n# Rolling File Appender\r\n#\r\n\r\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Logfile size and and 30-day backups\r\nlog4j.appender.RFA.MaxFileSize=256MB\r\ nlog4j.appender.RFA.MaxBackupIndex=10\r\n\r\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n# Custom Logging levels\r\n\r\nhadoop.metrics.log.level=INFO\r\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\r\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\r\n\r\n# Jets3t library\r\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\r\n\r\n#\r\n# Null Appender\r\n# Trap security logger on the hadoop client side\r\n#\r\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\r\n\r\n#\r\n# Event Counter Appender\r\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\r\n#\r\nlog4j.appender.EventCounter =org.apache.hadoop.log.metrics.EventCounter\r\n\r\n# Removes \"deprecated\" messages\r\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\r\n\r\n#\r\n# HDFS block state change log from block manager\r\n#\r\n# Uncomment the following to suppress normal block state change\r\n# messages from BlockManager in NameNode.\r\n#log4j.logger.BlockStateChange=WARN\r\n" - } - } - } - ], - "host_groups": [ - { - "name": "master", - "components": [ - { - "name": "APP_TIMELINE_SERVER" - }, - { - "name": "HISTORYSERVER" - }, - { - "name": "HBASE_REGIONSERVER" - }, - { - "name": "HBASE_CLIENT" - }, - { - "name": "WEBHCAT_SERVER" - }, - { - "name": "HCAT" - }, - { - "name": "NAMENODE" - }, - { - "name": "AMBARI_SERVER" - }, - { - "name": "HDFS_CLIENT" - }, - { - "name": "HIVE_CLIENT" - }, - { - "name": "NODEMANAGER" - }, - { - "name": "DATANODE" - }, - { - "name": "RESOURCEMANAGER" - }, - { - "name": "ZOOKEEPER_SERVER" - }, - { - "name": "ZOOKEEPER_CLIENT" - }, - { - "name": "HBASE_MASTER" - }, - { - "name": "HIVE_SERVER" - }, - { - "name": "SECONDARY_NAMENODE" - }, - { - "name": "HIVE_METASTORE" - }, - { - "name": "YARN_CLIENT" - }, - { - "name": "MAPREDUCE2_CLIENT" - }, - { - "name": "MYSQL_SERVER" - }, - { - "name": "GANGLIA_SERVER" - }, - { - "name" : "DRPC_SERVER" - }, - { - "name" : "STORM_UI_SERVER" - }, - { - "name" : "NIMBUS" - }, - { - "name" : "KAFKA_BROKER" - } - ], - "cardinality": "1" - }, - { - "name": "slave_1", - "components": [ - { - "name": "HBASE_CLIENT" - }, - { - "name": "HDFS_CLIENT" - }, - { - "name": "HIVE_CLIENT" - }, - { - "name": "NODEMANAGER" - }, - { - "name": "DATANODE" - }, - { - "name": "ZOOKEEPER_SERVER" - }, - { - "name": "ZOOKEEPER_CLIENT" - }, - { - "name": "YARN_CLIENT" - }, - { - "name": "MAPREDUCE2_CLIENT" - }, - { - "name" : "KAFKA_BROKER" - }, - { - "name": "GANGLIA_MONITOR" - }, - { - "name" : "SUPERVISOR" - } - ], - "cardinality": "1" - } - ], - "Blueprints": { - "blueprint_name": "hdp-multinode-eagle", - "stack_name": "HDP", - "stack_version": "2.2" -} -}
