bharathv commented on a change in pull request #1583: URL: https://github.com/apache/hbase/pull/1583#discussion_r416022329
########## File path: hbase-it/src/test/java/org/apache/hadoop/hbase/CoprocClusterManager.java ########## @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.Objects; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hbase.client.AsyncAdmin; +import org.apache.hadoop.hbase.client.AsyncConnection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecRequest; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecResponse; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecService; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Overrides commands to make use of Coproc where possible. + */ [email protected] +public class CoprocClusterManager extends HBaseClusterManager { + private static final Logger LOG = LoggerFactory.getLogger(CoprocClusterManager.class); + + @Override + protected Pair<Integer, String> exec(String hostname, ServiceType service, String... cmd) + throws IOException { + // we don't have ssh to log in as the service user, so wrap the command in sudo. + // assumes user running the hbase process has sudo privileges to target service. + final String serviceUser = getServiceUser(service); + final String[] commandPrefix = + !Objects.equals(serviceUser, getServiceUser(ServiceType.HBASE_MASTER)) Review comment: Okay. ########## File path: hbase-it/src/test/java/org/apache/hadoop/hbase/ShellExecCoproc.java ########## @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; +import com.google.protobuf.Service; +import java.io.IOException; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; +import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessor; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecRequest; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecResponse; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +import org.apache.hadoop.util.Shell; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * Receives shell commands from the client and executes them blindly. + */ [email protected] +public class ShellExecCoproc + extends ShellExecEndpoint.ShellExecService + implements MasterCoprocessor, RegionServerCoprocessor { + private static final Logger LOG = LoggerFactory.getLogger(ShellExecCoproc.class); + + public static final String ASYNC_DELAY_KEY = "hbase.it.shellexeccoproc.async.delay"; + public static final long DEFAULT_ASYNC_DELAY = 1_000; + + private final ExecutorService workerPool; + private Configuration conf; + + public ShellExecCoproc() { + workerPool = Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder() + .setNameFormat(ShellExecCoproc.class.getSimpleName() + "-{}") + .setDaemon(true) + .setUncaughtExceptionHandler((t, e) -> LOG.warn("Thread {} threw", t, e)) + .build()); + } + + @Override + public Iterable<Service> getServices() { + return Collections.singletonList(this); + } + + @Override + public void start(CoprocessorEnvironment env) { + conf = env.getConfiguration(); + } + + @Override + public void shellExec( + final RpcController controller, + final ShellExecRequest request, + final RpcCallback<ShellExecResponse> done + ) { + final Shell.ShellCommandExecutor shell = prepareShell(request, false); + + ShellExecResponse.Builder builder = ShellExecResponse.newBuilder(); + try { + builder = doExec(shell, builder); + } catch (IOException e) { + LOG.info("Failure launching process", e); + CoprocessorRpcUtils.setControllerException(controller, e); + } + + done.run(builder.build()); + } + + @Override + public void shellExecAsync( + final RpcController controller, + final ShellExecRequest request, + final RpcCallback<ShellExecResponse> done + ) { + final Shell.ShellCommandExecutor shell = prepareShell(request, true); + final long sleepDuration = conf.getLong(ASYNC_DELAY_KEY, DEFAULT_ASYNC_DELAY); + workerPool.submit(() -> { + try { + // sleep first so that the RPC can ACK. Review comment: Ah, the kill -9 for chaos monkey.. now I see it. The method name "shellExecAsync()" gave an impression that it is a generic shell exec utility. Also I think you added the "async" version just so that it acks the RPC and sends a response back. I think we should do one of the following 1. Since we already have a blocking version and async version is only for kill, a special command (IIUC) rename it to make it more self explanatory? 2. If you don't agree with above, add some comments why this async version exists? ########## File path: hbase-it/src/test/java/org/apache/hadoop/hbase/ShellExecCoproc.java ########## @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; +import com.google.protobuf.Service; +import java.io.IOException; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; +import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessor; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecRequest; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecResponse; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +import org.apache.hadoop.util.Shell; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * Receives shell commands from the client and executes them blindly. + */ [email protected] +public class ShellExecCoproc Review comment: I meant, we can condense that to fewer lines. ########## File path: hbase-it/src/test/java/org/apache/hadoop/hbase/ShellExecCoproc.java ########## @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; +import com.google.protobuf.Service; +import java.io.IOException; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; +import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessor; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecRequest; +import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ShellExecEndpoint.ShellExecResponse; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +import org.apache.hadoop.util.Shell; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * Receives shell commands from the client and executes them blindly. + */ [email protected] +public class ShellExecCoproc + extends ShellExecEndpoint.ShellExecService + implements MasterCoprocessor, RegionServerCoprocessor { + private static final Logger LOG = LoggerFactory.getLogger(ShellExecCoproc.class); + + public static final String ASYNC_DELAY_KEY = "hbase.it.shellexeccoproc.async.delay"; + public static final long DEFAULT_ASYNC_DELAY = 1_000; + + private final ExecutorService workerPool; + private Configuration conf; + + public ShellExecCoproc() { + workerPool = Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder() + .setNameFormat(ShellExecCoproc.class.getSimpleName() + "-{}") + .setDaemon(true) + .setUncaughtExceptionHandler((t, e) -> LOG.warn("Thread {} threw", t, e)) + .build()); + } + + @Override + public Iterable<Service> getServices() { + return Collections.singletonList(this); + } + + @Override + public void start(CoprocessorEnvironment env) { + conf = env.getConfiguration(); + } + + @Override + public void shellExec( + final RpcController controller, + final ShellExecRequest request, + final RpcCallback<ShellExecResponse> done + ) { + final Shell.ShellCommandExecutor shell = prepareShell(request, false); + + ShellExecResponse.Builder builder = ShellExecResponse.newBuilder(); + try { + builder = doExec(shell, builder); + } catch (IOException e) { + LOG.info("Failure launching process", e); + CoprocessorRpcUtils.setControllerException(controller, e); + } + + done.run(builder.build()); + } + + @Override + public void shellExecAsync( + final RpcController controller, + final ShellExecRequest request, + final RpcCallback<ShellExecResponse> done + ) { + final Shell.ShellCommandExecutor shell = prepareShell(request, true); + final long sleepDuration = conf.getLong(ASYNC_DELAY_KEY, DEFAULT_ASYNC_DELAY); + workerPool.submit(() -> { + try { + // sleep first so that the RPC can ACK. + Thread.sleep(sleepDuration); + doExec(shell, ShellExecResponse.newBuilder()); + } catch (InterruptedException e) { + LOG.info("Interrupted before launching process.", e); Review comment: sg ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
