[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r410080579
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GceAutoScaler.java
##
@@ -0,0 +1,539 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.NetworkInterface;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.base.Preconditions;
+import com.google.common.net.InetAddresses;
+import
org.apache.curator.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
Review comment:
That seems reasonable :+1:
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r410069896
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GceAutoScaler.java
##
@@ -0,0 +1,539 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.NetworkInterface;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.base.Preconditions;
+import com.google.common.net.InetAddresses;
+import
org.apache.curator.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
Review comment:
On further thought I think just leaving it as `gce` is fine, feel free to
ignore this comment.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE URL: https://github.com/apache/druid/pull/8987#discussion_r410062740 ## File path: extensions-contrib/gce-extensions/pom.xml ## @@ -0,0 +1,126 @@ + + +http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; +xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + +org.apache.druid +druid +0.18.0-SNAPSHOT Review comment: This needs to be `0.19.0-SNAPSHOT` This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r410065652
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GceAutoScaler.java
##
@@ -0,0 +1,539 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.NetworkInterface;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.base.Preconditions;
+import com.google.common.net.InetAddresses;
+import
org.apache.curator.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
Review comment:
One last thought, not really suggesting a change just thinking ahead, but do
you imagine there could ever be any other types of Google compute based
autoscalers that _do not_ use managed instance groups? I wonder if we should
name the type for this `gce-mig` or something like that to leave space for
variation. On the other hand, i guess the only other raw GCE based autoscaler
implementation I can imagine would work like the AWS autoscaler which just
deals directly with raw instances (i built one of these in the past as a
private extension), so maybe it isn't worth distinguishing. Thoughts?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r397047442
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r397043748
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GceAutoScaler.java
##
@@ -0,0 +1,539 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.NetworkInterface;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.base.Preconditions;
+import com.google.common.net.InetAddresses;
+import
org.apache.curator.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GceAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GceAutoScaler.class);
+
+ private final GceEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ private static final long POLL_INTERVAL_MS = 5 * 1000; // 5 sec
+ private static final int RUNNING_INSTANCES_MAX_RETRIES = 10;
+ private static final int OPERATION_END_MAX_RETRIES = 10;
+
+ @JsonCreator
+ public GceAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GceEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+Preconditions.checkArgument(minNumWorkers > 0,
+"minNumWorkers must be greater than 0");
+this.minNumWorkers = minNumWorkers;
+Preconditions.checkArgument(maxNumWorkers > 0,
+"maxNumWorkers must be greater than 0");
+Preconditions.checkArgument(maxNumWorkers > minNumWorkers,
+"maxNumWorkers must be greater than
minNumWorkers");
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ @VisibleForTesting
+ public GceAutoScaler(
+
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE URL: https://github.com/apache/druid/pull/8987#discussion_r378139096 ## File path: extensions-contrib/gce-extensions/pom.xml ## @@ -0,0 +1,121 @@ + + +http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; +xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + +org.apache.druid +druid +0.17.0-incubating-SNAPSHOT Review comment: Also, this needs to be `0.18.0-SNAPSHOT`, since there has been a new release and we have graduated from incubating, this might be related to why travis is failing This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378087243
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378111966
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378110552
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378125823
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEEnvironmentConfig.java
##
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ */
+public class GCEEnvironmentConfig
+{
+ /**
+ * numInstances: the number of workers to try to spawn at each call to
provision
+ * projectId: the id of the project where to operate
+ * zoneName: the name of the zone where to operata
+ * instanceTemplate: the template to use when creating the instances
+ * minworkers: the minimum number of workers in the pool (*)
+ * maxWorkers: the maximum number of workers in the pool (*)
+ *
+ * (*) both used by the caller of the AutoScaler to know if it makes sense
to call
+ * provision / terminate or if there is no hope that something would be
done
+ */
+ private final int numInstances;
+ private final String projectId;
+ private final String zoneName;
+ private final String managedInstanceGroupName;
+
+ @JsonCreator
+ public GCEEnvironmentConfig(
+ @JsonProperty("numInstances") int numInstances,
+ @JsonProperty("projectId") String projectId,
+ @JsonProperty("zoneName") String zoneName,
+ @JsonProperty("managedInstanceGroupName") String
managedInstanceGroupName
+ )
+ {
+this.numInstances = numInstances;
Review comment:
Should probably use `Preconditions.checkNotNull` to ensure these values are
not null
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378105136
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378108567
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE URL: https://github.com/apache/druid/pull/8987#discussion_r378126140 ## File path: extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEModule.java ## @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.autoscaling.gce; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; + +import java.util.Collections; +import java.util.List; + +public class GCEModule implements DruidModule Review comment: nit: I think this should be `GceModule` to be consistent with other stuff, e.g. `GcpModule` This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378108736
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378087959
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE URL: https://github.com/apache/druid/pull/8987#discussion_r378126388 ## File path: extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEUtils.java ## @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.autoscaling.gce; + +import java.util.Iterator; +import java.util.List; +import java.util.Locale; + +/** + * Simple collection of utilities extracted to ease testing and simplify the GCEAutoScaler class + */ +public class GCEUtils Review comment: same nit: I think this should be `GceUtils` This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378085600
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
Review comment:
we have a `@VisibleForTesting` annotation that might be appropriate to add
here
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378093159
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378133515
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378127666
##
File path:
extensions-contrib/gce-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScalerTest.java
##
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.databind.BeanProperty;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.InjectableValues;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.NetworkInterface;
+import com.google.api.services.compute.model.Operation;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ */
+public class GCEAutoScalerTest
+{
+ private Compute mockCompute = null;
+ // id -> ip & ip -> id
+ private Compute.Instances mockInstances = null;
+ private Compute.Instances.List mockIpToIdRequest = null;
+ private Compute.Instances.List mockIdToIpRequest = null;
+ // running instances
+ private Compute.InstanceGroupManagers mockInstanceGroupManagers = null;
+ private Compute.InstanceGroupManagers.ListManagedInstances
mockInstancesRequest = null;
+ // terminate
+ private Compute.InstanceGroupManagers.DeleteInstances mockDeleteRequest =
null;
+ //provision
+ private Compute.InstanceGroupManagers.Resize mockResizeRequest = null;
+
+ @Before
+ public void setUp()
+ {
+// for every test let's create all (only a subset needed for each test tho)
+
+mockCompute = EasyMock.createMock(Compute.class);
+
+mockInstances = EasyMock.createMock(Compute.Instances.class);
+mockIpToIdRequest = EasyMock.createMock(Compute.Instances.List.class);
+mockIdToIpRequest = EasyMock.createMock(Compute.Instances.List.class);
+
+mockInstanceGroupManagers =
EasyMock.createMock(Compute.InstanceGroupManagers.class);
+mockInstancesRequest = EasyMock.createMock(
+Compute.InstanceGroupManagers.ListManagedInstances.class
+);
+
+mockDeleteRequest =
EasyMock.createMock(Compute.InstanceGroupManagers.DeleteInstances.class);
+
+mockResizeRequest =
EasyMock.createMock(Compute.InstanceGroupManagers.Resize.class);
+ }
+
+ @After
+ public void tearDown()
+ {
+// not calling verify here as we use different bits and pieces in each test
+ }
+
+ private static void verifyAutoScaler(final GCEAutoScaler autoScaler)
+ {
+Assert.assertEquals(1, autoScaler.getEnvConfig().getNumInstances());
+Assert.assertEquals(4, autoScaler.getMaxNumWorkers());
+Assert.assertEquals(2, autoScaler.getMinNumWorkers());
+Assert.assertEquals("winkie-country",
autoScaler.getEnvConfig().getZoneName());
+Assert.assertEquals("super-project",
autoScaler.getEnvConfig().getProjectId());
+Assert.assertEquals("druid-mig",
autoScaler.getEnvConfig().getManagedInstanceGroupName());
+ }
+
+ @Test
+ public void testConfig()
+ {
+final String json = "{\n"
++ " \"envConfig\" : {\n"
++ " \"numInstances\" : 1,\n"
++ " \"projectId\" : \"super-project\",\n"
++ " \"zoneName\" : \"winkie-country\",\n"
+
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378096457
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378086798
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378092985
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378090407
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378084312
##
File path: extensions-contrib/gce-extensions/pom.xml
##
@@ -0,0 +1,121 @@
+
+
+http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";
xmlns="http://maven.apache.org/POM/4.0.0";
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";>
+
+org.apache.druid
+druid
+0.17.0-incubating-SNAPSHOT
+../../pom.xml
+
+ 4.0.0
+
+ org.apache.druid.extensions.contrib
+ gce-extensions
+ gce-extensions
+ Extension to support the autoscaling in GCE
+
+
+UTF-8
+
+
+
+ org.apache.druid
+ druid-core
+ ${project.parent.version}
+ provided
+
+
+ org.apache.druid
+ druid-indexing-service
+ ${project.parent.version}
+ provided
+
+
+ org.apache.druid
+ druid-aws-common
+ ${project.parent.version}
+ provided
+
+
+ org.apache.druid
+ druid-processing
+ ${project.parent.version}
+ provided
+
+
+ com.google.code.findbugs
+ jsr305
+ provided
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+ provided
+
+
+ com.google.guava
+ guava
+ provided
+
+
+ com.google.inject
+ guice
+ provided
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ provided
+
+
+ com.google.apis
+ google-api-services-compute
+ v1-rev214-1.25.0
+ compile
+
+
+ com.google.http-client
+ google-http-client
Review comment:
I believe `google-http-client`, `google-http-client-jackson2`, and
`google-api-client` could omit version and be marked `provided` since they will
be supplied by `druid-gcp-common`.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378104481
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378112981
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
[GitHub] [druid] clintropolis commented on a change in pull request #8987: Adding support for autoscaling in GCE
clintropolis commented on a change in pull request #8987: Adding support for
autoscaling in GCE
URL: https://github.com/apache/druid/pull/8987#discussion_r378092773
##
File path:
extensions-contrib/gce-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/gce/GCEAutoScaler.java
##
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.gce;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.compute.Compute;
+import com.google.api.services.compute.ComputeScopes;
+import com.google.api.services.compute.model.Instance;
+import
com.google.api.services.compute.model.InstanceGroupManagersDeleteInstancesRequest;
+import
com.google.api.services.compute.model.InstanceGroupManagersListManagedInstancesResponse;
+import com.google.api.services.compute.model.InstanceList;
+import com.google.api.services.compute.model.ManagedInstance;
+import com.google.api.services.compute.model.Operation;
+import com.google.common.net.InetAddresses;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import
org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
+import org.apache.druid.java.util.emitter.EmittingLogger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * This module permits the autoscaling of the workers in GCE
+ *
+ * General notes:
+ * - The IPs are IPs as in Internet Protocol, and they look like 1.2.3.4
+ * - The IDs are the names of the instances of instances created, they look
like prefix-abcd,
+ * where the prefix is chosen by you and abcd is a suffix assigned by GCE
+ */
+@JsonTypeName("gce")
+public class GCEAutoScaler implements AutoScaler
+{
+ private static final EmittingLogger log = new
EmittingLogger(GCEAutoScaler.class);
+
+ private final GCEEnvironmentConfig envConfig;
+ private final int minNumWorkers;
+ private final int maxNumWorkers;
+ private final SimpleWorkerProvisioningConfig config; // For future use
+
+ private Compute cachedComputeService = null;
+
+ @JsonCreator
+ public GCEAutoScaler(
+ @JsonProperty("minNumWorkers") int minNumWorkers,
+ @JsonProperty("maxNumWorkers") int maxNumWorkers,
+ @JsonProperty("envConfig") GCEEnvironmentConfig envConfig,
+ @JacksonInject SimpleWorkerProvisioningConfig config
+ )
+ {
+this.minNumWorkers = minNumWorkers;
+this.maxNumWorkers = maxNumWorkers;
+this.envConfig = envConfig;
+this.config = config;
+ }
+
+ /**
+ * CAVEAT: this is meant to be used only for testing passing a mock version
of Compute
+ */
+ public GCEAutoScaler(
+ int minNumWorkers,
+ int maxNumWorkers,
+ GCEEnvironmentConfig envConfig,
+ SimpleWorkerProvisioningConfig config,
+ Compute compute
+ )
+ {
+this(minNumWorkers, maxNumWorkers, envConfig, config);
+this.cachedComputeService = compute;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMinNumWorkers()
+ {
+return minNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public int getMaxNumWorkers()
+ {
+return maxNumWorkers;
+ }
+
+ @Override
+ @JsonProperty
+ public GCEEnvironmentConfig getEnvConfig()
+ {
+return envConfig;
+ }
+
+ private synchronized Compute createComputeService()
+ throws IOException, GeneralSecurityException, InterruptedException
+ {
+final int max_retries = 5;
+final long retries_interval = 5 * 1000; // 5 secs.
+
+int retries = 0;
+while (cachedCo
