This is an automated email from the ASF dual-hosted git repository.
jihoonson pushed a commit to branch 0.21.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/0.21.0 by this push:
new bfc066c Revert "Add support for Blacklisting some domains for
HTTPInputSource (#10535)" (#10871) (#10872)
bfc066c is described below
commit bfc066cdf577c0a8e44c8b0186931a749128e5cf
Author: Jihoon Son <[email protected]>
AuthorDate: Fri Feb 12 15:27:30 2021 -0800
Revert "Add support for Blacklisting some domains for HTTPInputSource
(#10535)" (#10871) (#10872)
This reverts commit 6b14bdb3a53d6aec45e485e6849956a69720ba3f.
---
.../druid/data/input/impl/HttpInputSource.java | 15 +--
.../data/input/impl/HttpInputSourceConfig.java | 112 ---------------------
.../druid/data/input/impl/HttpInputSourceTest.java | 74 +-------------
docs/configuration/index.md | 9 --
.../druid/metadata/input/InputSourceModule.java | 3 -
.../metadata/input/InputSourceModuleTest.java | 76 +-------------
website/.spelling | 2 -
7 files changed, 4 insertions(+), 287 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
index 36d6b97..21480fd 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
@@ -19,7 +19,6 @@
package org.apache.druid.data.input.impl;
-import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
@@ -29,7 +28,6 @@ import org.apache.druid.data.input.InputRowSchema;
import org.apache.druid.data.input.InputSourceReader;
import org.apache.druid.data.input.InputSplit;
import org.apache.druid.data.input.SplitHintSpec;
-import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.metadata.PasswordProvider;
import javax.annotation.Nullable;
@@ -48,25 +46,17 @@ public class HttpInputSource extends AbstractInputSource
implements SplittableIn
@Nullable
private final PasswordProvider httpAuthenticationPasswordProvider;
- private final HttpInputSourceConfig config;
-
@JsonCreator
public HttpInputSource(
@JsonProperty("uris") List<URI> uris,
@JsonProperty("httpAuthenticationUsername") @Nullable String
httpAuthenticationUsername,
- @JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider
httpAuthenticationPasswordProvider,
- @JacksonInject HttpInputSourceConfig config
+ @JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider
httpAuthenticationPasswordProvider
)
{
Preconditions.checkArgument(uris != null && !uris.isEmpty(), "Empty URIs");
- uris.forEach(uri -> Preconditions.checkArgument(
- config.isURIAllowed(uri),
- StringUtils.format("Access to [%s] DENIED!", uri)
- ));
this.uris = uris;
this.httpAuthenticationUsername = httpAuthenticationUsername;
this.httpAuthenticationPasswordProvider =
httpAuthenticationPasswordProvider;
- this.config = config;
}
@JsonProperty
@@ -107,8 +97,7 @@ public class HttpInputSource extends AbstractInputSource
implements SplittableIn
return new HttpInputSource(
Collections.singletonList(split.get()),
httpAuthenticationUsername,
- httpAuthenticationPasswordProvider,
- config
+ httpAuthenticationPasswordProvider
);
}
diff --git
a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java
b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java
deleted file mode 100644
index a84fb31..0000000
---
a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.data.input.impl;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Preconditions;
-
-import java.net.URI;
-import java.util.List;
-import java.util.Objects;
-
-public class HttpInputSourceConfig
-{
- @JsonProperty
- private final List<String> allowListDomains;
- @JsonProperty
- private final List<String> denyListDomains;
-
- @JsonCreator
- public HttpInputSourceConfig(
- @JsonProperty("allowListDomains") List<String> allowListDomains,
- @JsonProperty("denyListDomains") List<String> denyListDomains
- )
- {
- this.allowListDomains = allowListDomains;
- this.denyListDomains = denyListDomains;
- Preconditions.checkArgument(
- this.denyListDomains == null || this.allowListDomains == null,
- "Can only use one of allowList or blackList"
- );
- }
-
- public List<String> getAllowListDomains()
- {
- return allowListDomains;
- }
-
- public List<String> getDenyListDomains()
- {
- return denyListDomains;
- }
-
- private static boolean matchesAny(List<String> domains, URI uri)
- {
- for (String domain : domains) {
- if (uri.getHost().endsWith(domain)) {
- return true;
- }
- }
- return false;
- }
-
- public boolean isURIAllowed(URI uri)
- {
- if (allowListDomains != null) {
- return matchesAny(allowListDomains, uri);
- }
- if (denyListDomains != null) {
- return !matchesAny(denyListDomains, uri);
- }
- // No blacklist/allowList configured, all URLs are allowed
- return true;
- }
-
- @Override
- public String toString()
- {
- return "HttpInputSourceConfig{" +
- "allowListDomains=" + allowListDomains +
- ", denyListDomains=" + denyListDomains +
- '}';
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- HttpInputSourceConfig config = (HttpInputSourceConfig) o;
- return Objects.equals(allowListDomains, config.allowListDomains) &&
- Objects.equals(denyListDomains, config.denyListDomains);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(allowListDomains, denyListDomains);
- }
-}
-
diff --git
a/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
b/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
index 892fb38..61be2cc 100644
---
a/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
+++
b/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
@@ -19,7 +19,6 @@
package org.apache.druid.data.input.impl;
-import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import org.apache.druid.data.input.InputSource;
@@ -29,91 +28,20 @@ import org.junit.Test;
import java.io.IOException;
import java.net.URI;
-import java.util.Collections;
public class HttpInputSourceTest
{
@Test
public void testSerde() throws IOException
{
- HttpInputSourceConfig httpInputSourceConfig = new HttpInputSourceConfig(
- null,
- null
- );
final ObjectMapper mapper = new ObjectMapper();
- mapper.setInjectableValues(new InjectableValues.Std().addValue(
- HttpInputSourceConfig.class,
- httpInputSourceConfig
- ));
final HttpInputSource source = new HttpInputSource(
ImmutableList.of(URI.create("http://test.com/http-test";)),
"myName",
- new DefaultPasswordProvider("myPassword"),
- httpInputSourceConfig
+ new DefaultPasswordProvider("myPassword")
);
final byte[] json = mapper.writeValueAsBytes(source);
final HttpInputSource fromJson = (HttpInputSource) mapper.readValue(json,
InputSource.class);
Assert.assertEquals(source, fromJson);
}
-
- @Test(expected = IllegalArgumentException.class)
- public void testDenyListDomainThrowsException()
- {
- new HttpInputSource(
- ImmutableList.of(URI.create("http://deny.com/http-test";)),
- "myName",
- new DefaultPasswordProvider("myPassword"),
- new HttpInputSourceConfig(null, Collections.singletonList("deny.com"))
- );
- }
-
- @Test
- public void testDenyListDomainNoMatch()
- {
- new HttpInputSource(
- ImmutableList.of(URI.create("http://allow.com/http-test";)),
- "myName",
- new DefaultPasswordProvider("myPassword"),
- new HttpInputSourceConfig(null, Collections.singletonList("deny.com"))
- );
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testAllowListDomainThrowsException()
- {
- new HttpInputSource(
- ImmutableList.of(URI.create("http://deny.com/http-test";)),
- "myName",
- new DefaultPasswordProvider("myPassword"),
- new HttpInputSourceConfig(Collections.singletonList("allow.com"), null)
- );
- }
-
- @Test
- public void testAllowListDomainMatch()
- {
- new HttpInputSource(
- ImmutableList.of(URI.create("http://allow.com/http-test";)),
- "myName",
- new DefaultPasswordProvider("myPassword"),
- new HttpInputSourceConfig(Collections.singletonList("allow.com"), null)
- );
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testEmptyAllowListDomainMatch()
- {
- new HttpInputSource(
- ImmutableList.of(URI.create("http://allow.com/http-test";)),
- "myName",
- new DefaultPasswordProvider("myPassword"),
- new HttpInputSourceConfig(Collections.emptyList(), null)
- );
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testCannotSetBothAllowAndDenyList()
- {
- new HttpInputSourceConfig(Collections.singletonList("allow.com"),
Collections.singletonList("deny.com"));
- }
}
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index f9b1705..622da78 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -1355,15 +1355,6 @@ The amount of direct memory needed by Druid is at least
ensure at least this amount of direct memory is available by providing
`-XX:MaxDirectMemorySize=<VALUE>` at the command
line.
-#### Indexer Security Configuration
-You can optionally configure following additional configs to restrict druid
ingestion
-
-|Property|Possible Values|Description|Default|
-|--------|---------------|-----------|-------|
-|`druid.ingestion.http.allowListDomains`|List of domains|Allowed domains from
which ingestion will be allowed. Only one of allowList or denyList can be
set.|empty list|
-|`druid.ingestion.http.denyListDomains`|List of domains|Blacklisted domains
from which ingestion will NOT be allowed. Only one of allowList or denyList can
be set. |empty list|
-
-
#### Query Configurations
See [general query configuration](#general-query-configuration).
diff --git
a/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java
b/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java
index eb612f7..0423af4 100644
---
a/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java
+++
b/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java
@@ -24,8 +24,6 @@ import com.fasterxml.jackson.databind.jsontype.NamedType;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
-import org.apache.druid.data.input.impl.HttpInputSourceConfig;
-import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.initialization.DruidModule;
import java.util.List;
@@ -49,6 +47,5 @@ public class InputSourceModule implements DruidModule
@Override
public void configure(Binder binder)
{
- JsonConfigProvider.bind(binder, "druid.ingestion.http",
HttpInputSourceConfig.class);
}
}
diff --git
a/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java
b/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java
index 81fa5d3..67126b0 100644
---
a/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java
+++
b/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java
@@ -25,27 +25,12 @@ import com.fasterxml.jackson.databind.cfg.MapperConfig;
import com.fasterxml.jackson.databind.introspect.AnnotatedClass;
import com.fasterxml.jackson.databind.introspect.AnnotatedClassResolver;
import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.ProvisionException;
-import org.apache.druid.data.input.impl.HttpInputSourceConfig;
-import org.apache.druid.guice.DruidGuiceExtensions;
-import org.apache.druid.guice.JsonConfigurator;
-import org.apache.druid.guice.LazySingleton;
-import org.apache.druid.guice.LifecycleModule;
-import org.apache.druid.guice.ServerModule;
-import org.apache.druid.jackson.JacksonModule;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import javax.validation.Validation;
-import javax.validation.Validator;
-import java.util.Collections;
import java.util.List;
-import java.util.Properties;
import java.util.stream.Collectors;
public class InputSourceModuleTest
@@ -56,8 +41,7 @@ public class InputSourceModuleTest
@Before
public void setUp()
{
- InputSourceModule inputSourceModule = new InputSourceModule();
- for (Module jacksonModule : inputSourceModule.getJacksonModules()) {
+ for (Module jacksonModule : new InputSourceModule().getJacksonModules()) {
mapper.registerModule(jacksonModule);
}
}
@@ -75,62 +59,4 @@ public class InputSourceModuleTest
Assert.assertNotNull(subtypes);
Assert.assertEquals(SQL_NAMED_TYPE, Iterables.getOnlyElement(subtypes));
}
-
- @Test
- public void testHttpInputSourceAllowConfig()
- {
- Properties props = new Properties();
- props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]");
- Injector injector = makeInjectorWithProperties(props);
- HttpInputSourceConfig instance =
injector.getInstance(HttpInputSourceConfig.class);
- Assert.assertEquals(new
HttpInputSourceConfig(Collections.singletonList("allow.com"), null), instance);
- }
-
- @Test
- public void testHttpInputSourceDenyConfig()
- {
- Properties props = new Properties();
- props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]");
- Injector injector = makeInjectorWithProperties(props);
- HttpInputSourceConfig instance =
injector.getInstance(HttpInputSourceConfig.class);
- Assert.assertEquals(new HttpInputSourceConfig(null,
Collections.singletonList("deny.com")), instance);
- }
-
- @Test(expected = ProvisionException.class)
- public void testHttpInputSourceBothAllowDenyConfig()
- {
- Properties props = new Properties();
- props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]");
- props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]");
- Injector injector = makeInjectorWithProperties(props);
- injector.getInstance(HttpInputSourceConfig.class);
- }
-
- @Test
- public void testHttpInputSourceDefaultConfig()
- {
- Properties props = new Properties();
- Injector injector = makeInjectorWithProperties(props);
- HttpInputSourceConfig instance =
injector.getInstance(HttpInputSourceConfig.class);
- Assert.assertEquals(new HttpInputSourceConfig(null, null), instance);
- Assert.assertNull(instance.getAllowListDomains());
- Assert.assertNull(instance.getDenyListDomains());
- }
-
- private Injector makeInjectorWithProperties(final Properties props)
- {
- return Guice.createInjector(
- ImmutableList.of(
- new DruidGuiceExtensions(),
- new LifecycleModule(),
- new ServerModule(),
- binder -> {
-
binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator());
- binder.bind(JsonConfigurator.class).in(LazySingleton.class);
- binder.bind(Properties.class).toInstance(props);
- },
- new JacksonModule(),
- new InputSourceModule()
- ));
- }
}
diff --git a/website/.spelling b/website/.spelling
index 4184c67..3ca5e9f 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -1667,7 +1667,6 @@ _default_tier
addr
affinityConfig
allowAll
-allowList
ANDed
array_mod
autoscale
@@ -1681,7 +1680,6 @@ cpuacct
dataSourceName
datetime
defaultHistory
-denyList
doubleMax
doubleMin
doubleSum
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
