[
https://issues.apache.org/jira/browse/CARBONDATA-2956?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Kunal Kapoor resolved CARBONDATA-2956.
--------------------------------------
Resolution: Fixed
> CarbonReader can't support use configuration to read S3 data
> ------------------------------------------------------------
>
> Key: CARBONDATA-2956
> URL: https://issues.apache.org/jira/browse/CARBONDATA-2956
> Project: CarbonData
> Issue Type: Bug
> Components: other
> Affects Versions: 1.5.0
> Reporter: xubo245
> Assignee: xubo245
> Priority: Major
> Fix For: 1.5.0
>
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
> Test code:
> {code:java}
> package org.apache.carbondata.examples.sdk;
> import org.apache.carbondata.common.logging.LogService;
> import org.apache.carbondata.common.logging.LogServiceFactory;
> import org.apache.carbondata.core.metadata.datatype.DataTypes;
> import org.apache.carbondata.core.scan.expression.ColumnExpression;
> import org.apache.carbondata.core.scan.expression.LiteralExpression;
> import
> org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
> import org.apache.carbondata.sdk.file.*;
> import org.apache.hadoop.conf.Configuration;
> import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
> import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
> import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
> /**
> * Example for testing CarbonWriter on S3
> */
> public class SDKS3Example {
> public static void main(String[] args) throws Exception {
> LogService logger =
> LogServiceFactory.getLogService(SDKS3Example.class.getName());
> if (args == null || args.length < 3) {
> logger.error("Usage: java CarbonS3Example: <access-key>
> <secret-key>"
> + "<s3-endpoint> [table-path-on-s3] [rows]");
> System.exit(0);
> }
> String path = "s3a://sdk/WriterOutput";
> if (args.length > 3) {
> path=args[3];
> }
> int num = 3;
> if (args.length > 4) {
> num = Integer.parseInt(args[4]);
> }
> Configuration conf = new Configuration();
> conf.set(ACCESS_KEY,args[0]);
> conf.set(SECRET_KEY,args[1]);
> conf.set(ENDPOINT,args[2]);
> // Field[] fields = new Field[2];
> // fields[0] = new Field("name", DataTypes.STRING);
> // fields[1] = new Field("age", DataTypes.INT);
> // CarbonWriterBuilder builder = CarbonWriter.builder()
> // .setAccessKey(args[0])
> // .setSecretKey(args[1])
> // .setEndPoint(args[2])
> // .outputPath(path);
> //
> // CarbonWriter writer = builder.buildWriterForCSVInput(new
> Schema(fields), conf);
> //
> // for (int i = 0; i < num; i++) {
> // writer.write(new String[]{"robot" + (i % 10),
> String.valueOf(i)});
> // }
> // writer.close();
> // Read data
> EqualToExpression equalToExpression = new EqualToExpression(
> new ColumnExpression("name", DataTypes.STRING),
> new LiteralExpression("robot1", DataTypes.STRING));
> CarbonReader reader = CarbonReader
> .builder(path, "_temp")
> .projection(new String[]{"name", "age"})
> .filter(equalToExpression)
> .build(conf);
> System.out.println("\nData:");
> int i = 0;
> while (i < 20 && reader.hasNext()) {
> Object[] row = (Object[]) reader.readNextRow();
> System.out.println(row[0] + " " + row[1]);
> i++;
> }
> System.out.println("\nFinished");
> reader.close();
> }
> }
> {code}
> Exception:
> {code:java}
> log4j:WARN No appenders could be found for logger
> (org.apache.hadoop.util.Shell).
> log4j:WARN Please initialize the log4j system properly.
> log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more
> info.
> Exception in thread "main" com.amazonaws.AmazonClientException: Unable to
> load AWS credentials from any provider in the chain
> at
> com.amazonaws.auth.AWSCredentialsProviderChain.getCredentials(AWSCredentialsProviderChain.java:117)
> at
> com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3521)
> at
> com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1031)
> at
> com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:994)
> at
> org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:297)
> at
> org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653)
> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92)
> at
> org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687)
> at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669)
> at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371)
> at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
> at
> org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:74)
> at
> org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:66)
> at
> org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile.<init>(HDFSCarbonFile.java:41)
> at
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.<init>(S3CarbonFile.java:41)
> at
> org.apache.carbondata.core.datastore.impl.DefaultFileTypeProvider.getCarbonFile(DefaultFileTypeProvider.java:53)
> at
> org.apache.carbondata.core.datastore.impl.FileFactory.getCarbonFile(FileFactory.java:99)
> at
> org.apache.carbondata.core.metadata.schema.table.CarbonTable.buildTable(CarbonTable.java:241)
> at
> org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:191)
> at
> org.apache.carbondata.examples.sdk.SDKS3Example.main(SDKS3Example.java:91)
> Process finished with exit code 1
> {code}
> if configure like:
> {code:java}
> CarbonReader reader = CarbonReader
> .builder(path, "_temp")
> .projection(new String[]{"name", "age"})
> .filter(equalToExpression)
> .setAccessKey(args[0])
> .setSecretKey(args[1])
> .setEndPoint(args[2])
> .build(new Configuration(false));
> {code}
>
> Exception2:
> {code:java}
> log4j:WARN No appenders could be found for logger
> (org.apache.hadoop.util.Shell).
> log4j:WARN Please initialize the log4j system properly.
> log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more
> info.
> Exception in thread "main" com.amazonaws.AmazonClientException: Unable to
> execute HTTP request: sdk.obs.cn-north-1.myhwclouds.com: nodename nor
> servname provided, or not known
> at
> com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:454)
> at
> com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:232)
> at
> com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3528)
> at
> com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1031)
> at
> com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:994)
> at
> org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:297)
> at
> org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653)
> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92)
> at
> org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687)
> at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669)
> at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371)
> at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
> at
> org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:74)
> at
> org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:66)
> at
> org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile.<init>(HDFSCarbonFile.java:41)
> at
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.<init>(S3CarbonFile.java:41)
> at
> org.apache.carbondata.core.datastore.impl.DefaultFileTypeProvider.getCarbonFile(DefaultFileTypeProvider.java:53)
> at
> org.apache.carbondata.core.datastore.impl.FileFactory.getCarbonFile(FileFactory.java:99)
> at
> org.apache.carbondata.core.metadata.schema.table.CarbonTable.buildTable(CarbonTable.java:241)
> at
> org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:191)
> at
> org.apache.carbondata.examples.sdk.SDKS3Example.main(SDKS3Example.java:91)
> Caused by: java.net.UnknownHostException: sdk.obs.cn-north-1.myhwclouds.com:
> nodename nor servname provided, or not known
> at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
> at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928)
> at
> java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
> at java.net.InetAddress.getAllByName0(InetAddress.java:1276)
> at java.net.InetAddress.getAllByName(InetAddress.java:1192)
> at java.net.InetAddress.getAllByName(InetAddress.java:1126)
> at
> org.apache.http.impl.conn.SystemDefaultDnsResolver.resolve(SystemDefaultDnsResolver.java:45)
> at
> org.apache.http.impl.conn.DefaultClientConnectionOperator.resolveHostname(DefaultClientConnectionOperator.java:278)
> at
> org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:162)
> at
> org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
> at
> org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:641)
> at
> org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
> at
> org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
> at
> org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
> at
> com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:384)
> ... 20 more
> Process finished with exit code 1
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
