This is an automated email from the ASF dual-hosted git repository.
nishant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 6b14bdb Add support for Blacklisting some domains for HTTPInputSource
(#10535)
6b14bdb is described below
commit 6b14bdb3a53d6aec45e485e6849956a69720ba3f
Author: Nishant Bangarwa <[email protected]>
AuthorDate: Mon Nov 2 21:47:25 2020 +0530
Add support for Blacklisting some domains for HTTPInputSource (#10535)
fix inspections
refactor class name
change name
add allowList as well
distinguish between empty and null list
Fix CI
---
.../druid/data/input/impl/HttpInputSource.java | 15 ++-
.../data/input/impl/HttpInputSourceConfig.java | 112 +++++++++++++++++++++
.../druid/data/input/impl/HttpInputSourceTest.java | 74 +++++++++++++-
docs/configuration/index.md | 9 ++
.../druid/metadata/input/InputSourceModule.java | 3 +
.../metadata/input/InputSourceModuleTest.java | 76 +++++++++++++-
website/.spelling | 2 +
7 files changed, 287 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
index 21480fd..36d6b97 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
@@ -19,6 +19,7 @@
package org.apache.druid.data.input.impl;
+import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
@@ -28,6 +29,7 @@ import org.apache.druid.data.input.InputRowSchema;
import org.apache.druid.data.input.InputSourceReader;
import org.apache.druid.data.input.InputSplit;
import org.apache.druid.data.input.SplitHintSpec;
+import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.metadata.PasswordProvider;
import javax.annotation.Nullable;
@@ -46,17 +48,25 @@ public class HttpInputSource extends AbstractInputSource
implements SplittableIn
@Nullable
private final PasswordProvider httpAuthenticationPasswordProvider;
+ private final HttpInputSourceConfig config;
+
@JsonCreator
public HttpInputSource(
@JsonProperty("uris") List<URI> uris,
@JsonProperty("httpAuthenticationUsername") @Nullable String
httpAuthenticationUsername,
- @JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider
httpAuthenticationPasswordProvider
+ @JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider
httpAuthenticationPasswordProvider,
+ @JacksonInject HttpInputSourceConfig config
)
{
Preconditions.checkArgument(uris != null && !uris.isEmpty(), "Empty URIs");
+ uris.forEach(uri -> Preconditions.checkArgument(
+ config.isURIAllowed(uri),
+ StringUtils.format("Access to [%s] DENIED!", uri)
+ ));
this.uris = uris;
this.httpAuthenticationUsername = httpAuthenticationUsername;
this.httpAuthenticationPasswordProvider =
httpAuthenticationPasswordProvider;
+ this.config = config;
}
@JsonProperty
@@ -97,7 +107,8 @@ public class HttpInputSource extends AbstractInputSource
implements SplittableIn
return new HttpInputSource(
Collections.singletonList(split.get()),
httpAuthenticationUsername,
- httpAuthenticationPasswordProvider
+ httpAuthenticationPasswordProvider,
+ config
);
}
diff --git
a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java
b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java
new file mode 100644
index 0000000..a84fb31
--- /dev/null
+++
b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.data.input.impl;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Objects;
+
+public class HttpInputSourceConfig
+{
+ @JsonProperty
+ private final List<String> allowListDomains;
+ @JsonProperty
+ private final List<String> denyListDomains;
+
+ @JsonCreator
+ public HttpInputSourceConfig(
+ @JsonProperty("allowListDomains") List<String> allowListDomains,
+ @JsonProperty("denyListDomains") List<String> denyListDomains
+ )
+ {
+ this.allowListDomains = allowListDomains;
+ this.denyListDomains = denyListDomains;
+ Preconditions.checkArgument(
+ this.denyListDomains == null || this.allowListDomains == null,
+ "Can only use one of allowList or blackList"
+ );
+ }
+
+ public List<String> getAllowListDomains()
+ {
+ return allowListDomains;
+ }
+
+ public List<String> getDenyListDomains()
+ {
+ return denyListDomains;
+ }
+
+ private static boolean matchesAny(List<String> domains, URI uri)
+ {
+ for (String domain : domains) {
+ if (uri.getHost().endsWith(domain)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public boolean isURIAllowed(URI uri)
+ {
+ if (allowListDomains != null) {
+ return matchesAny(allowListDomains, uri);
+ }
+ if (denyListDomains != null) {
+ return !matchesAny(denyListDomains, uri);
+ }
+ // No blacklist/allowList configured, all URLs are allowed
+ return true;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "HttpInputSourceConfig{" +
+ "allowListDomains=" + allowListDomains +
+ ", denyListDomains=" + denyListDomains +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ HttpInputSourceConfig config = (HttpInputSourceConfig) o;
+ return Objects.equals(allowListDomains, config.allowListDomains) &&
+ Objects.equals(denyListDomains, config.denyListDomains);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(allowListDomains, denyListDomains);
+ }
+}
+
diff --git
a/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
b/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
index 61be2cc..892fb38 100644
---
a/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
+++
b/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
@@ -19,6 +19,7 @@
package org.apache.druid.data.input.impl;
+import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import org.apache.druid.data.input.InputSource;
@@ -28,20 +29,91 @@ import org.junit.Test;
import java.io.IOException;
import java.net.URI;
+import java.util.Collections;
public class HttpInputSourceTest
{
@Test
public void testSerde() throws IOException
{
+ HttpInputSourceConfig httpInputSourceConfig = new HttpInputSourceConfig(
+ null,
+ null
+ );
final ObjectMapper mapper = new ObjectMapper();
+ mapper.setInjectableValues(new InjectableValues.Std().addValue(
+ HttpInputSourceConfig.class,
+ httpInputSourceConfig
+ ));
final HttpInputSource source = new HttpInputSource(
ImmutableList.of(URI.create("http://test.com/http-test";)),
"myName",
- new DefaultPasswordProvider("myPassword")
+ new DefaultPasswordProvider("myPassword"),
+ httpInputSourceConfig
);
final byte[] json = mapper.writeValueAsBytes(source);
final HttpInputSource fromJson = (HttpInputSource) mapper.readValue(json,
InputSource.class);
Assert.assertEquals(source, fromJson);
}
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testDenyListDomainThrowsException()
+ {
+ new HttpInputSource(
+ ImmutableList.of(URI.create("http://deny.com/http-test";)),
+ "myName",
+ new DefaultPasswordProvider("myPassword"),
+ new HttpInputSourceConfig(null, Collections.singletonList("deny.com"))
+ );
+ }
+
+ @Test
+ public void testDenyListDomainNoMatch()
+ {
+ new HttpInputSource(
+ ImmutableList.of(URI.create("http://allow.com/http-test";)),
+ "myName",
+ new DefaultPasswordProvider("myPassword"),
+ new HttpInputSourceConfig(null, Collections.singletonList("deny.com"))
+ );
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testAllowListDomainThrowsException()
+ {
+ new HttpInputSource(
+ ImmutableList.of(URI.create("http://deny.com/http-test";)),
+ "myName",
+ new DefaultPasswordProvider("myPassword"),
+ new HttpInputSourceConfig(Collections.singletonList("allow.com"), null)
+ );
+ }
+
+ @Test
+ public void testAllowListDomainMatch()
+ {
+ new HttpInputSource(
+ ImmutableList.of(URI.create("http://allow.com/http-test";)),
+ "myName",
+ new DefaultPasswordProvider("myPassword"),
+ new HttpInputSourceConfig(Collections.singletonList("allow.com"), null)
+ );
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testEmptyAllowListDomainMatch()
+ {
+ new HttpInputSource(
+ ImmutableList.of(URI.create("http://allow.com/http-test";)),
+ "myName",
+ new DefaultPasswordProvider("myPassword"),
+ new HttpInputSourceConfig(Collections.emptyList(), null)
+ );
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCannotSetBothAllowAndDenyList()
+ {
+ new HttpInputSourceConfig(Collections.singletonList("allow.com"),
Collections.singletonList("deny.com"));
+ }
}
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 756debd..a4047d8 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -1353,6 +1353,15 @@ The amount of direct memory needed by Druid is at least
ensure at least this amount of direct memory is available by providing
`-XX:MaxDirectMemorySize=<VALUE>` at the command
line.
+#### Indexer Security Configuration
+You can optionally configure following additional configs to restrict druid
ingestion
+
+|Property|Possible Values|Description|Default|
+|--------|---------------|-----------|-------|
+|`druid.ingestion.http.allowListDomains`|List of domains|Allowed domains from
which ingestion will be allowed. Only one of allowList or denyList can be
set.|empty list|
+|`druid.ingestion.http.denyListDomains`|List of domains|Blacklisted domains
from which ingestion will NOT be allowed. Only one of allowList or denyList can
be set. |empty list|
+
+
#### Query Configurations
See [general query configuration](#general-query-configuration).
diff --git
a/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java
b/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java
index 0423af4..eb612f7 100644
---
a/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java
+++
b/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java
@@ -24,6 +24,8 @@ import com.fasterxml.jackson.databind.jsontype.NamedType;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
+import org.apache.druid.data.input.impl.HttpInputSourceConfig;
+import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.initialization.DruidModule;
import java.util.List;
@@ -47,5 +49,6 @@ public class InputSourceModule implements DruidModule
@Override
public void configure(Binder binder)
{
+ JsonConfigProvider.bind(binder, "druid.ingestion.http",
HttpInputSourceConfig.class);
}
}
diff --git
a/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java
b/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java
index 67126b0..81fa5d3 100644
---
a/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java
+++
b/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java
@@ -25,12 +25,27 @@ import com.fasterxml.jackson.databind.cfg.MapperConfig;
import com.fasterxml.jackson.databind.introspect.AnnotatedClass;
import com.fasterxml.jackson.databind.introspect.AnnotatedClassResolver;
import com.fasterxml.jackson.databind.jsontype.NamedType;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.ProvisionException;
+import org.apache.druid.data.input.impl.HttpInputSourceConfig;
+import org.apache.druid.guice.DruidGuiceExtensions;
+import org.apache.druid.guice.JsonConfigurator;
+import org.apache.druid.guice.LazySingleton;
+import org.apache.druid.guice.LifecycleModule;
+import org.apache.druid.guice.ServerModule;
+import org.apache.druid.jackson.JacksonModule;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import javax.validation.Validation;
+import javax.validation.Validator;
+import java.util.Collections;
import java.util.List;
+import java.util.Properties;
import java.util.stream.Collectors;
public class InputSourceModuleTest
@@ -41,7 +56,8 @@ public class InputSourceModuleTest
@Before
public void setUp()
{
- for (Module jacksonModule : new InputSourceModule().getJacksonModules()) {
+ InputSourceModule inputSourceModule = new InputSourceModule();
+ for (Module jacksonModule : inputSourceModule.getJacksonModules()) {
mapper.registerModule(jacksonModule);
}
}
@@ -59,4 +75,62 @@ public class InputSourceModuleTest
Assert.assertNotNull(subtypes);
Assert.assertEquals(SQL_NAMED_TYPE, Iterables.getOnlyElement(subtypes));
}
+
+ @Test
+ public void testHttpInputSourceAllowConfig()
+ {
+ Properties props = new Properties();
+ props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]");
+ Injector injector = makeInjectorWithProperties(props);
+ HttpInputSourceConfig instance =
injector.getInstance(HttpInputSourceConfig.class);
+ Assert.assertEquals(new
HttpInputSourceConfig(Collections.singletonList("allow.com"), null), instance);
+ }
+
+ @Test
+ public void testHttpInputSourceDenyConfig()
+ {
+ Properties props = new Properties();
+ props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]");
+ Injector injector = makeInjectorWithProperties(props);
+ HttpInputSourceConfig instance =
injector.getInstance(HttpInputSourceConfig.class);
+ Assert.assertEquals(new HttpInputSourceConfig(null,
Collections.singletonList("deny.com")), instance);
+ }
+
+ @Test(expected = ProvisionException.class)
+ public void testHttpInputSourceBothAllowDenyConfig()
+ {
+ Properties props = new Properties();
+ props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]");
+ props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]");
+ Injector injector = makeInjectorWithProperties(props);
+ injector.getInstance(HttpInputSourceConfig.class);
+ }
+
+ @Test
+ public void testHttpInputSourceDefaultConfig()
+ {
+ Properties props = new Properties();
+ Injector injector = makeInjectorWithProperties(props);
+ HttpInputSourceConfig instance =
injector.getInstance(HttpInputSourceConfig.class);
+ Assert.assertEquals(new HttpInputSourceConfig(null, null), instance);
+ Assert.assertNull(instance.getAllowListDomains());
+ Assert.assertNull(instance.getDenyListDomains());
+ }
+
+ private Injector makeInjectorWithProperties(final Properties props)
+ {
+ return Guice.createInjector(
+ ImmutableList.of(
+ new DruidGuiceExtensions(),
+ new LifecycleModule(),
+ new ServerModule(),
+ binder -> {
+
binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator());
+ binder.bind(JsonConfigurator.class).in(LazySingleton.class);
+ binder.bind(Properties.class).toInstance(props);
+ },
+ new JacksonModule(),
+ new InputSourceModule()
+ ));
+ }
}
diff --git a/website/.spelling b/website/.spelling
index 096b300..f4decbc 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -1645,6 +1645,7 @@ _default_tier
addr
affinityConfig
allowAll
+allowList
ANDed
array_mod
autoscale
@@ -1658,6 +1659,7 @@ cpuacct
dataSourceName
datetime
defaultHistory
+denyList
doubleMax
doubleMin
doubleSum
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
