steveloughran commented on a change in pull request #1208: HADOOP-16423.
S3Guard fsck: Check metadata consistency between S3 and metadatastore (log)
URL: https://github.com/apache/hadoop/pull/1208#discussion_r321808553
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
##########
@@ -1485,6 +1486,93 @@ private void vprintln(PrintStream out, String format,
Object...
}
}
+ /**
+ * Prune metadata that has not been modified recently.
+ */
+ static class Fsck extends S3GuardTool {
+ public static final String CHECK_FLAG = "check";
+
+ public static final String NAME = "fsck";
+ public static final String PURPOSE = "Compares S3 with MetadataStore, and "
+ + "returns a failure status if any rules or invariants are violated. "
+ + "Only works with DynamoDbMetadataStore.";
+ private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
+ "\t" + PURPOSE + "\n\n" +
+ "Common options:\n" +
+ " " + CHECK_FLAG + " Check the metadata store for errors, but do "
+ + "not fix any issues.\n";
+
+ Fsck(Configuration conf) {
+ super(conf, CHECK_FLAG);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public String getUsage() {
+ return USAGE;
+ }
+
+ public int run(String[] args, PrintStream out) throws
+ InterruptedException, IOException {
+ List<String> paths = parseArgs(args);
+ if (paths.isEmpty()) {
+ out.println(USAGE);
+ throw invalidArgs("no arguments");
+ }
+
+ String s3Path = paths.get(0);
+ try {
+ initS3AFileSystem(s3Path);
+ } catch (Exception e) {
+ errorln("Failed to initialize S3AFileSystem from path: " + s3Path);
+ throw e;
+ }
+
+ URI uri = toUri(s3Path);
+ Path root;
+ if (uri.getPath().isEmpty()) {
+ root = new Path("/");
+ } else {
+ root = new Path(uri.getPath());
+ }
+
+ final S3AFileSystem fs = getFilesystem();
+ initMetadataStore(false);
+ final MetadataStore ms = getStore();
+
+ if (ms == null ||
+ !(ms instanceof DynamoDBMetadataStore)) {
+ errorln(s3Path + " path uses MS: " + ms);
+ errorln(NAME + " can be only used with a DynamoDB backed s3a bucket.");
+ errorln(USAGE);
+ return ERROR;
+ }
+
+ final CommandFormat commandFormat = getCommandFormat();
+ if (commandFormat.getOpt(CHECK_FLAG)) {
+ // do the check
+ S3GuardFsck s3GuardFsck = new S3GuardFsck(fs, ms);
+ try {
+ s3GuardFsck.compareS3ToMs(fs.qualify(root));
+ } catch (IOException e) {
+ errorln("Error while running the check: compareS3ToMs");
Review comment:
Is this needed; the runner logs anyway?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
