[hadoop] branch trunk updated: YARN-10233. Fix YARN UI2 Daemon Logs
This is an automated email from the ASF dual-hosted git repository.
prabhujoseph pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 94f7470 YARN-10233. Fix YARN UI2 Daemon Logs
94f7470 is described below
commit 94f7470c71104af1871a22c34fbbce14a60c33ab
Author: Prabhu Joseph
AuthorDate: Wed Apr 15 13:39:33 2020 +0530
YARN-10233. Fix YARN UI2 Daemon Logs
Contributed by Akhil PB.
---
.../hadoop-yarn-ui/src/main/webapp/.gitignore | 1 +
.../src/main/webapp/app/models/yarn-rm-log.js | 3 ++-
.../src/main/webapp/app/serializers/yarn-rm-log.js | 28 --
.../src/main/webapp/app/templates/yarn-tools.hbs | 2 +-
.../app/templates/yarn-tools/yarn-rm-log.hbs | 10
5 files changed, 25 insertions(+), 19 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
index 338997f..082fb27 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
@@ -2,3 +2,4 @@ tmp/
node_modules/
bower_components/
dist/
+node/
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-log.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-log.js
index 2b6febf..17adc9f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-log.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-log.js
@@ -19,5 +19,6 @@
import DS from 'ember-data';
export default DS.Model.extend({
- logfileName: DS.attr()
+ logFileName: DS.attr(),
+ logFileUrl: DS.attr()
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-log.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-log.js
index da834f1..900e0ff 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-log.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-log.js
@@ -19,8 +19,8 @@
import DS from 'ember-data';
export default DS.JSONAPISerializer.extend({
- normalizeResponse(store, primaryModelClass, payload, id, requestType) {
-const pattern = new RegExp('', 'g');
+ normalizeResponse(store, primaryModelClass, payload) {
+const pattern = new RegExp('|', 'g');
let fileNames = payload.match(pattern);
if (fileNames == null) {
@@ -29,16 +29,20 @@ export default DS.JSONAPISerializer.extend({
let logfileNames = [];
for (var i = 0; i < fileNames.length; i++) {
- var fileName = fileNames[i].match(//);
-if (fileName.length != null) {
- logfileNames.push({
-id: i,
-type: primaryModelClass.modelName,
-attributes: {
- logfileName: fileName[1]
-}
- });
-}
+ var fileNameMatch = fileNames[i].match(/.+<\/A>|.+<\/a>/);
+ var logFileUrl = fileNameMatch[1] || fileNameMatch[2];
+ var logFileName = logFileUrl.replace('logs', '').replace(/\//g, '');
+
+ if (fileNameMatch.length != null) {
+logfileNames.push({
+ id: i,
+ type: primaryModelClass.modelName,
+ attributes: {
+logFileUrl: logFileUrl,
+logFileName: logFileName
+ }
+});
+ }
}
return { data : logfileNames };
},
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-tools.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-tools.hbs
index 3efcf57..2306e4b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-tools.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-tools.hbs
@@ -52,7 +52,7 @@
{{/link-to}}
{{/link-to}}
{{#link-to 'yarn-tools.yarn-rm-log' tagName="li"}}
-{{#link-to 'yarn-tools.yarn-rm-log'}}YARN Daemon logs
+{{#link-to 'yarn-tools.yarn-rm-log'}}YARN Daemon Logs
{{/link-to}}
{{/link-to}}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-tools/yarn-rm-log.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-tools/yarn-rm-log.hbs
index 3cf536d..6d80246 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-tools/yarn-rm-log.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-tools/yarn-rm-log.hbs
@@ -17,15 +17,15 @@
}}
[hadoop] branch branch-3.2 updated: YARN-9995. Code cleanup in TestSchedConfCLI. Contributed by Bilwa S T.
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new a7eb95f YARN-9995. Code cleanup in TestSchedConfCLI. Contributed by
Bilwa S T.
a7eb95f is described below
commit a7eb95f1149278e4517a8a0e62e76bcd4fded960
Author: Szilard Nemeth
AuthorDate: Wed Apr 15 08:05:07 2020 +0200
YARN-9995. Code cleanup in TestSchedConfCLI. Contributed by Bilwa S T.
---
.../hadoop/yarn/client/cli/TestSchedConfCLI.java | 142 ++---
1 file changed, 69 insertions(+), 73 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
index 8a6c9c2..5ff7438 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
@@ -65,6 +65,7 @@ import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -73,17 +74,10 @@ import static org.junit.Assert.assertTrue;
*/
public class TestSchedConfCLI extends JerseyTestBase {
- private ByteArrayOutputStream sysOutStream;
- private PrintStream sysOut;
-
- private ByteArrayOutputStream sysErrStream;
- private PrintStream sysErr;
-
private SchedConfCLI cli;
private static MockRM rm;
private static String userName;
- private static CapacitySchedulerConfiguration csConf;
private static final File CONF_FILE = new File(new File("target",
"test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
@@ -100,14 +94,6 @@ public class TestSchedConfCLI extends JerseyTestBase {
@Before
public void setUp() {
-sysOutStream = new ByteArrayOutputStream();
-sysOut = new PrintStream(sysOutStream);
-System.setOut(sysOut);
-
-sysErrStream = new ByteArrayOutputStream();
-sysErr = new PrintStream(sysErrStream);
-System.setErr(sysErr);
-
cli = new SchedConfCLI();
}
@@ -130,8 +116,8 @@ public class TestSchedConfCLI extends JerseyTestBase {
+ ioe.getMessage(), ioe);
}
- csConf = new CapacitySchedulerConfiguration(new Configuration(false),
- false);
+ CapacitySchedulerConfiguration csConf = new
+ CapacitySchedulerConfiguration(new Configuration(false), false);
setupQueueConfiguration(csConf);
try {
@@ -245,51 +231,46 @@ public class TestSchedConfCLI extends JerseyTestBase {
@Test(timeout = 1)
public void testInvalidConf() throws Exception {
+ByteArrayOutputStream sysErrStream = new ByteArrayOutputStream();
+PrintStream sysErr = new PrintStream(sysErrStream);
+System.setErr(sysErr);
+
// conf pair with no key should be invalid
-int exitCode = cli.run(new String[] {"-add", "root.a:=confVal"});
-assertTrue("Should return an error code", exitCode != 0);
-assertTrue(sysErrStream.toString().contains("Specify configuration key " +
-"value as confKey=confVal."));
-exitCode = cli.run(new String[] {"-update", "root.a:=confVal"});
-assertTrue("Should return an error code", exitCode != 0);
-assertTrue(sysErrStream.toString().contains("Specify configuration key " +
-"value as confKey=confVal."));
-
-exitCode = cli.run(new String[] {"-add", "root.a:confKey=confVal=conf"});
-assertTrue("Should return an error code", exitCode != 0);
-assertTrue(sysErrStream.toString().contains("Specify configuration key " +
-"value as confKey=confVal."));
-exitCode = cli.run(new String[] {"-update", "root.a:confKey=confVal=c"});
-assertTrue("Should return an error code", exitCode != 0);
-assertTrue(sysErrStream.toString().contains("Specify configuration key " +
-"value as confKey=confVal."));
+executeCommand(sysErrStream, "-add", "root.a:=confVal");
+executeCommand(sysErrStream, "-update", "root.a:=confVal");
+executeCommand(sysErrStream, "-add", "root.a:confKey=confVal=conf");
+executeCommand(sysErrStream, "-update", "root.a:confKey=confVal=c");
+ }
+
+ private void executeCommand(ByteArrayOutputStream sysErrStream, String op,
+ String queueConf) throws Exception {
+int exitCode = cli.run(new String[] {op, queueConf});
+assertNotEquals("Should return an error code", 0, exitCode);
+assertTrue(sysErrStream.toString()
+.contains("Specify configuration key " + "value as confKey=confVal."));
}
@Test(timeout =
[hadoop] branch branch-3.2 updated: YARN-9354. Resources should be created with ResourceTypesTestHelper instead of TestUtils. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 58e559b YARN-9354. Resources should be created with
ResourceTypesTestHelper instead of TestUtils. Contributed by Andras Gyori
58e559b is described below
commit 58e559b5ac1d54f6e0263e72d486a753e1e1fd9a
Author: Szilard Nemeth
AuthorDate: Wed Apr 15 08:16:15 2020 +0200
YARN-9354. Resources should be created with ResourceTypesTestHelper instead
of TestUtils. Contributed by Andras Gyori
---
.../resourcetypes/ResourceTypesTestHelper.java | 24 ++
.../TestApplicationMasterService.java | 22 ++--
...estCapacitySchedulerWithMultiResourceTypes.java | 19 +
.../scheduler/capacity/TestUtils.java | 19 -
4 files changed, 36 insertions(+), 48 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
index 3c3c2cc..198b395 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
@@ -57,14 +57,15 @@ public final class ResourceTypesTestHelper {
Resource resource = RECORD_FACTORY.newRecordInstance(Resource.class);
resource.setMemorySize(memory);
resource.setVirtualCores(vCores);
-
-for (Map.Entry customResource :
-customResources.entrySet()) {
- String resourceName = customResource.getKey();
- ResourceInformation resourceInformation =
- createResourceInformation(resourceName,
- customResource.getValue());
- resource.setResourceInformation(resourceName, resourceInformation);
+if (customResources != null) {
+ for (Map.Entry customResource :
+ customResources.entrySet()) {
+String resourceName = customResource.getKey();
+ResourceInformation resourceInformation =
+createResourceInformation(resourceName,
+customResource.getValue());
+resource.setResourceInformation(resourceName, resourceInformation);
+ }
}
return resource;
}
@@ -107,7 +108,12 @@ public final class ResourceTypesTestHelper {
public static Map extractCustomResourcesAsStrings(
Resource res) {
Map resValues = extractCustomResources(res);
-return resValues.entrySet().stream()
+return convertCustomResources(resValues);
+ }
+
+ public static Map convertCustomResources(
+ Map customResources) {
+return customResources.entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey, e -> String.valueOf(e.getValue(;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index bfe908d..45c5ac8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -82,7 +82,6 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
@@ -746,8 +745,8 @@ public class TestApplicationMasterService {
MockRM rm = new MockRM(yarnConf);
rm.start();
-MockNM nm1 = rm.registerNode("199.99.99.1:1234", TestUtils
-.createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+MockNM nm1 = rm.registerNode("199.99.99.1:1234", ResourceTypesTestHelper
+.newResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
[hadoop] branch branch-3.2 updated: YARN-10002. Code cleanup and improvements in ConfigurationStoreBaseTest. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 2f01a91 YARN-10002. Code cleanup and improvements in
ConfigurationStoreBaseTest. Contributed by Benjamin Teke
2f01a91 is described below
commit 2f01a91428341248620374271f7d362e4229a924
Author: Szilard Nemeth
AuthorDate: Wed Apr 15 08:24:15 2020 +0200
YARN-10002. Code cleanup and improvements in ConfigurationStoreBaseTest.
Contributed by Benjamin Teke
---
.../conf/FSSchedulerConfigurationStore.java| 7 ++
.../capacity/conf/InMemoryConfigurationStore.java | 7 ++
.../capacity/conf/LeveldbConfigurationStore.java | 1 +
.../capacity/conf/YarnConfigurationStore.java | 8 ++
.../capacity/conf/ZKConfigurationStore.java| 1 +
.../capacity/conf/ConfigurationStoreBaseTest.java | 40 +++---
.../conf/PersistentConfigurationStoreBaseTest.java | 136
.../conf/TestFSSchedulerConfigurationStore.java| 115 +
.../conf/TestLeveldbConfigurationStore.java| 112 ++---
.../capacity/conf/TestZKConfigurationStore.java| 140 -
10 files changed, 299 insertions(+), 268 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index 464ef14..690a60f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -21,6 +21,7 @@ package
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -327,6 +328,12 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
}
@Override
+ protected LinkedList getLogs() {
+// Unimplemented.
+return null;
+ }
+
+ @Override
protected Version getConfStoreVersion() throws Exception {
return null;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index 47dd6bd..8bd4ad0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -85,6 +86,12 @@ public class InMemoryConfigurationStore extends
YarnConfigurationStore {
}
@Override
+ protected LinkedList getLogs() {
+// Unimplemented.
+return null;
+ }
+
+ @Override
public Version getConfStoreVersion() throws Exception {
// Does nothing.
return null;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 5da002e..f921a9a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++
[hadoop] reference refs/remotes/origin/branch-3.3 updated (b397a3a -> 4bd37f2)
This is an automated email from the ASF dual-hosted git repository. prabhujoseph pushed a change to reference refs/remotes/origin/branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git. from b397a3a MAPREDUCE-7272. TaskAttemptListenerImpl excessive log messages. Contributed by Ahmed Hussein (ahussein) add 68a9562 HADOOP-16941. ITestS3GuardOutOfBandOperations.testListingDelete failing on versioned bucket (#1919) add 6bae8c4 HADOOP-16855. Changing wildfly dependency scope in hadoop-azure to compile add 7aad965 HADOOP-16982. Update Netty to 4.1.48.Final. Contributed by Lisheng Sun. add f473473 YARN-10234. FS-CS converter: don't enable auto-create queue property for root. Contributed by Peter Bacsko add 4bd37f2 YARN-5277. When localizers fail due to resource timestamps being out, provide more diagnostics. Contributed by Siddharth Ahuja No new revisions were added by this update. Summary of changes: hadoop-project/pom.xml | 14 +++- .../fs/s3a/ITestS3GuardOutOfBandOperations.java| 11 +++- hadoop-tools/hadoop-azure/pom.xml | 2 +- .../org/apache/hadoop/yarn/util/FSDownload.java| 10 ++- .../apache/hadoop/yarn/util/TestFSDownload.java| 74 ++ .../scheduler/fair/converter/FSQueueConverter.java | 4 +- .../converter/TestFSConfigToCSConfigConverter.java | 23 +-- .../fair/converter/TestFSQueueConverter.java | 2 +- 8 files changed, 126 insertions(+), 14 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9999. TestFSSchedulerConfigurationStore: Extend from ConfigurationStoreBaseTest, general code cleanup. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4db598e YARN-. TestFSSchedulerConfigurationStore: Extend from
ConfigurationStoreBaseTest, general code cleanup. Contributed by Benjamin Teke
4db598e is described below
commit 4db598e0e61bb13baf9ec7f854c0435cf5c71aa2
Author: Szilard Nemeth
AuthorDate: Wed Apr 15 08:37:22 2020 +0200
YARN-. TestFSSchedulerConfigurationStore: Extend from
ConfigurationStoreBaseTest, general code cleanup. Contributed by Benjamin Teke
---
.../capacity/conf/ConfigurationStoreBaseTest.java | 21 ++-
.../conf/PersistentConfigurationStoreBaseTest.java | 4 +
.../conf/TestFSSchedulerConfigurationStore.java| 162 +
3 files changed, 91 insertions(+), 96 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
index 3a8c362..2c36362 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
@@ -78,13 +78,24 @@ public abstract class ConfigurationStoreBaseTest {
confStore.close();
}
- YarnConfigurationStore.LogMutation prepareLogMutation(String key,
-String value)
+ YarnConfigurationStore.LogMutation prepareLogMutation(String... values)
throws Exception {
-Map update = new HashMap<>();
-update.put(key, value);
+Map updates = new HashMap<>();
+String key;
+String value;
+
+if (values.length % 2 != 0) {
+ throw new IllegalArgumentException("The number of parameters should be "
+
+"even.");
+}
+
+for (int i = 1; i <= values.length; i += 2) {
+ key = values[i - 1];
+ value = values[i];
+ updates.put(key, value);
+}
YarnConfigurationStore.LogMutation mutation =
-new YarnConfigurationStore.LogMutation(update, TEST_USER);
+new YarnConfigurationStore.LogMutation(updates, TEST_USER);
confStore.logMutation(mutation);
return mutation;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/PersistentConfigurationStoreBaseTest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/PersistentConfigurationStoreBaseTest.java
index 169c36d..5037921 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/PersistentConfigurationStoreBaseTest.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/PersistentConfigurationStoreBaseTest.java
@@ -25,6 +25,7 @@ import java.util.LinkedList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assume.assumeFalse;
/**
* Base class for the persistent {@link YarnConfigurationStore}
@@ -94,6 +95,9 @@ public abstract class PersistentConfigurationStoreBaseTest
extends
@Test
public void testMaxLogs() throws Exception {
+assumeFalse("test should be skipped for TestFSSchedulerConfigurationStore",
+ this instanceof TestFSSchedulerConfigurationStore);
+
conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2);
confStore.initialize(conf, schedConf, rmContext);
LinkedList logs = confStore.getLogs();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
index f5da5ac..5897741 100644
---
[hadoop] branch branch-3.1 updated: HADOOP-16982. Update Netty to 4.1.48.Final. Contributed by Lisheng Sun.
This is an automated email from the ASF dual-hosted git repository. iwasakims pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new 796bafe HADOOP-16982. Update Netty to 4.1.48.Final. Contributed by Lisheng Sun. 796bafe is described below commit 796bafedbaa032847bd18ca8b452a540baf674cf Author: Masatake Iwasaki AuthorDate: Thu Apr 16 11:57:08 2020 +0900 HADOOP-16982. Update Netty to 4.1.48.Final. Contributed by Lisheng Sun. --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 9647945..adf6e73 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -787,7 +787,7 @@ io.netty netty-all -4.1.45.Final +4.1.48.Final - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: HADOOP-16982. Update Netty to 4.1.48.Final. Contributed by Lisheng Sun.
This is an automated email from the ASF dual-hosted git repository. iwasakims pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 9d66a0e HADOOP-16982. Update Netty to 4.1.48.Final. Contributed by Lisheng Sun. 9d66a0e is described below commit 9d66a0e28e89c35a9376720fe1f7f821fbe3b44b Author: Masatake Iwasaki AuthorDate: Thu Apr 16 11:08:20 2020 +0900 HADOOP-16982. Update Netty to 4.1.48.Final. Contributed by Lisheng Sun. --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 351a07f..6e7d9d9 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -133,7 +133,7 @@ 4.1.0-incubating 3.2.4 3.10.6.Final -4.1.45.Final +4.1.48.Final 0.5.1 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-16985. Handle release package related issues (#1957)
This is an automated email from the ASF dual-hosted git repository.
vinayakumarb pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4d24d99 HADOOP-16985. Handle release package related issues (#1957)
4d24d99 is described below
commit 4d24d99e854e1718270a75807f7779a623758247
Author: Vinayakumar B
AuthorDate: Tue Apr 14 18:01:46 2020 +0530
HADOOP-16985. Handle release package related issues (#1957)
---
dev-support/bin/create-release | 4 +++-
hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml | 1 +
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index f4851d1..39a5d0d 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -651,10 +651,12 @@ function signartifacts
big_console_header "Signing the release"
- for i in ${ARTIFACTS_DIR}/*; do
+ run cd "${ARTIFACTS_DIR}"
+ for i in *; do
${GPG} --use-agent --armor --output "${i}.asc" --detach-sig "${i}"
sha512sum --tag "${i}" > "${i}.sha512"
done
+ run cd "${BASEDIR}"
if [[ "${ASFRELEASE}" = true ]]; then
echo "Fetching the Apache Hadoop KEYS file..."
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
index 7c725d7..87169420 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
@@ -56,6 +56,7 @@
**/build/**
**/file:/**
**/SecurityAuth.audit*
+patchprocess/**
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HADOOP-16465 listLocatedStatus() optimisation (#1943)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 94da630 HADOOP-16465 listLocatedStatus() optimisation (#1943)
94da630 is described below
commit 94da630cd2d1e056393f83acdac2747cf4b12bad
Author: Mukund Thakur
AuthorDate: Tue Apr 14 21:49:51 2020 +0530
HADOOP-16465 listLocatedStatus() optimisation (#1943)
Contributed by Mukund Thakur
Optimize S3AFileSystem.listLocatedStatus() to perform list
operations directly and then fallback to head checks for files
Change-Id: Ia2c0fa6fcc5967c49b914b92f41135d07dab0464
---
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java| 87 ++
.../hadoop/fs/s3a/ITestS3AFileOperationCost.java | 57 ++
2 files changed, 114 insertions(+), 30 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 9bd33a4..9630a9e 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -4283,42 +4283,69 @@ public class S3AFileSystem extends FileSystem
implements StreamCapabilities,
RemoteIterator iterator =
once("listLocatedStatus", path.toString(),
() -> {
-// lookup dir triggers existence check
-final S3AFileStatus fileStatus =
-(S3AFileStatus) getFileStatus(path);
-if (fileStatus.isFile()) {
- // simple case: File
- LOG.debug("Path is a file");
- return new Listing.SingleStatusRemoteIterator(
- filter.accept(path) ? toLocatedFileStatus(fileStatus) :
null);
-} else {
- // directory: trigger a lookup
- final String key = maybeAddTrailingSlash(pathToKey(path));
- final Listing.FileStatusAcceptor acceptor =
- new Listing.AcceptAllButSelfAndS3nDirs(path);
- boolean allowAuthoritative = allowAuthoritative(f);
- DirListingMetadata meta =
- S3Guard.listChildrenWithTtl(metadataStore, path,
- ttlTimeProvider, allowAuthoritative);
- final RemoteIterator cachedFileStatusIterator =
- listing.createProvidedFileStatusIterator(
- S3Guard.dirMetaToStatuses(meta), filter, acceptor);
- return (allowAuthoritative && meta != null
- && meta.isAuthoritative())
- ? listing.createLocatedFileStatusIterator(
- cachedFileStatusIterator)
- : listing.createLocatedFileStatusIterator(
- listing.createFileStatusListingIterator(path,
- createListObjectsRequest(key, "/"),
- filter,
- acceptor,
- cachedFileStatusIterator));
+// Assuming the path to be a directory,
+// trigger a list call directly.
+final RemoteIterator
+locatedFileStatusIteratorForDir =
+getLocatedFileStatusIteratorForDir(path, filter);
+
+// If no listing is present then path might be a file.
+if (!locatedFileStatusIteratorForDir.hasNext()) {
+ final S3AFileStatus fileStatus =
+ (S3AFileStatus) getFileStatus(path);
+ if (fileStatus.isFile()) {
+// simple case: File
+LOG.debug("Path is a file");
+return new Listing.SingleStatusRemoteIterator(
+filter.accept(path)
+? toLocatedFileStatus(fileStatus)
+: null);
+ }
}
+// Either empty or non-empty directory.
+return locatedFileStatusIteratorForDir;
});
return toLocatedFileStatusIterator(iterator);
}
/**
+ * Generate list located status for a directory.
+ * Also performing tombstone reconciliation for guarded directories.
+ * @param dir directory to check.
+ * @param filter a path filter.
+ * @return an iterator that traverses statuses of the given dir.
+ * @throws IOException in case of failure.
+ */
+ private RemoteIterator
getLocatedFileStatusIteratorForDir(
+ Path dir, PathFilter filter) throws IOException {
+final String key = maybeAddTrailingSlash(pathToKey(dir));
+final Listing.FileStatusAcceptor acceptor =
+new Listing.AcceptAllButSelfAndS3nDirs(dir);
+boolean allowAuthoritative = allowAuthoritative(dir);
+DirListingMetadata meta =
+
[hadoop] branch trunk updated: HDFS-15277. Parent directory in the explorer does not support all path formats. Contributed by Jianfei Jiang.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new cc5c1da HDFS-15277. Parent directory in the explorer does not support
all path formats. Contributed by Jianfei Jiang.
cc5c1da is described below
commit cc5c1da7c1c29618b5df785d9f1d7a0b737eced1
Author: Ayush Saxena
AuthorDate: Wed Apr 15 23:01:58 2020 +0530
HDFS-15277. Parent directory in the explorer does not support all path
formats. Contributed by Jianfei Jiang.
---
.../hadoop-hdfs-rbf/src/main/webapps/router/explorer.js| 14 +-
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 10 +++---
2 files changed, 16 insertions(+), 8 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js
index d2f3729..6917b70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js
@@ -300,6 +300,11 @@
}
function browse_directory(dir) {
+if (dir.match('^/+$')) {
+ $('#parentDir').prop('disabled', true);
+} else {
+ $('#parentDir').prop('disabled', false);
+}
var HELPERS = {
'helper_date_tostring' : function (chunk, ctx, bodies, params) {
var value = dust.helpers.tap(params.value, chunk, ctx);
@@ -378,11 +383,10 @@
$('#parentDir').click(function () {
- var current = current_directory;
- var lastIndex = current.lastIndexOf('/');
- var parent = current.substr(0, lastIndex);
- browse_directory(parent);
- })
+var current = current_directory;
+var parent = current.replace(/\/+[^/]+\/*$/,"") || '/';
+browse_directory(parent);
+ });
function init() {
dust.loadSource(dust.compile($('#tmpl-explorer').html(), 'explorer'));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
index f1a5f4a..cc55567 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
@@ -300,6 +300,11 @@
}
function browse_directory(dir) {
+if (dir.match('^/+$')) {
+ $('#parentDir').prop('disabled', true);
+} else {
+ $('#parentDir').prop('disabled', false);
+}
var HELPERS = {
'helper_date_tostring' : function (chunk, ctx, bodies, params) {
var value = dust.helpers.tap(params.value, chunk, ctx);
@@ -378,10 +383,9 @@
$('#parentDir').click(function () {
var current = current_directory;
-var lastIndex = current.lastIndexOf('/');
-var parent = current.substr(0, lastIndex);
+var parent = current.replace(/\/+[^/]+\/*$/,"") || '/';
browse_directory(parent);
- })
+ });
function init() {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HADOOP-16985. Handle release package related issues (#1957)
This is an automated email from the ASF dual-hosted git repository.
vinayakumarb pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new b4ba9be HADOOP-16985. Handle release package related issues (#1957)
b4ba9be is described below
commit b4ba9bed7c591ee858fa6139060c984913e8e94f
Author: Vinayakumar B
AuthorDate: Tue Apr 14 18:01:46 2020 +0530
HADOOP-16985. Handle release package related issues (#1957)
(cherry picked from commit 4d24d99e854e1718270a75807f7779a623758247)
---
dev-support/bin/create-release | 4 +++-
hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml | 1 +
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index f4851d1..39a5d0d 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -651,10 +651,12 @@ function signartifacts
big_console_header "Signing the release"
- for i in ${ARTIFACTS_DIR}/*; do
+ run cd "${ARTIFACTS_DIR}"
+ for i in *; do
${GPG} --use-agent --armor --output "${i}.asc" --detach-sig "${i}"
sha512sum --tag "${i}" > "${i}.sha512"
done
+ run cd "${BASEDIR}"
if [[ "${ASFRELEASE}" = true ]]; then
echo "Fetching the Apache Hadoop KEYS file..."
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
index 7c725d7..87169420 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
@@ -56,6 +56,7 @@
**/build/**
**/file:/**
**/SecurityAuth.audit*
+patchprocess/**
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
