Hello,
I am trying to create a cluster with HDFS programmatically by targeting the
REST APIs in the Ambari Server directly instead of using the Web frontend. I
set the HDFS service state to "INSTALLED" and I can see my host components
entering the state "INSTALLING", but after a while, they transition to
"INSTALL_FAILED". The problem is that neither the ambari-server.log/out nor the
ambari-agent.log/out have any error messages.
I'm not sure if there are any other sources of information I should be looking
at for errors, or if I'm doing anything wrong when calling the REST APIs (all
the API calls return successfully, though). My test is a NodeJS program which I
have attached in case anyone wants to take a look at it. The ambari.js code is
a supporting object model that enables easy calls to the REST API.
Does anyone know how can investigate why the state of my host components is
ending up in INSTALL_FAILED?
Thanks,
Ximo.
________________________________
Este mensaje se dirige exclusivamente a su destinatario. Puede consultar
nuestra pol?tica de env?o y recepci?n de correo electr?nico en el enlace
situado m?s abajo.
This message is intended exclusively for its addressee. We only send and
receive email on the basis of the terms set out at:
http://www.tid.es/ES/PAGINAS/disclaimer.aspx
;'use strict';
var Q = require('q');
var http = require('http');
var PATH_PREFIX = '/api/v1/';
var performRequest = function (request_options) {
var result = Q.defer();
var body = request_options.body;
delete request_options.body;
var req = http.request(request_options, function(res) {
var body = '';
res.setEncoding('utf8');
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
if (body) {
res.body = body;
try {
var json = JSON.parse(body);
res.body = json;
} catch(ignored) { }
}
if (Math.floor(res.statusCode / 100) === 2) result.resolve(res);
else result.reject(res);
});
});
if (body) req.write(JSON.stringify(body));
req.end();
return result.promise;
};
var isString = function(obj) {
return Object.prototype.toString.call(obj) == '[object String]';
};
var Service = function(cluster, serviceInfo) {
var service = this;
this.cluster = cluster;
this.name = serviceInfo.ServiceInfo.service_name;
this.state = serviceInfo.ServiceInfo.state;
this.componentNames = serviceInfo.components.map(function(componentInfo) {
return componentInfo.ServiceComponentInfo.component_name;
});
this.desiredConfigs = serviceInfo.ServiceInfo.desired_configs;
this.addComponent = function(componentName) {
var request_options = service.cluster.server.getBaseRequest();
request_options.method = 'POST';
request_options.path = PATH_PREFIX + 'clusters/' + service.cluster.name + '/services/' + service.name + '/components/' + componentName;
return performRequest(request_options).then(function(res) {
return componentName;
});
};
this.updateCluster = function() { throw new Error('Not implemented'); };
this.removeComponent = function() { throw new Error('Not implemented'); };
this.applyConfiguration = function(type, tag) {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'PUT';
request_options.path = PATH_PREFIX + 'clusters/' + service.cluster.name + '/services/' + service.name;
request_options.body = {
config: {}
};
request_options.body.config[type] = tag;
return performRequest(request_options);
};
this.install = function() {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'PUT';
request_options.path = PATH_PREFIX + 'clusters/' + service.cluster.name + '/services/' + service.name;
request_options.body = {
ServiceInfo: {
state: 'INSTALLED'
}
};
return performRequest(request_options);
};
this.start = function() {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'PUT';
request_options.path = PATH_PREFIX + 'clusters/' + service.cluster.name + '/services/' + service.name;
request_options.body = {
ServiceInfo: {
state: 'STARTED'
}
};
return performRequest(request_options);
};
return this;
};
var Host = function(cluster, hostInfo) {
var host = this;
this.cluster = cluster;
this.name = hostInfo.Hosts.host_name;
this.ip = hostInfo.Hosts.ip;
this.status = hostInfo.Hosts.host_status;
this.addComponents = function(components) {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'POST';
request_options.path = PATH_PREFIX + 'clusters/' + cluster.name + '/hosts?Hosts/host_name=' + host.name;
request_options.body = {
"host_components": components.map(function(component) {
return { HostRoles: {
"component_name": component
}};
})
};
return performRequest(request_options);
};
this.addComponent = function(component) {
return host.addComponents([component]);
};
};
var Cluster = function(server, clusterInfo) {
var cluster = this;
this.server = server;
this.name = clusterInfo.Clusters.cluster_name;
this.serviceNames = clusterInfo.services.map(function(service) {
return service.ServiceInfo.service_name;
});
this.hostNames = clusterInfo.hosts.map(function(host) {
return host.Hosts.host_name;
});
this.configurations = clusterInfo.configurations.map(function(config) {
return {
type: config.type,
tag: config.tag
};
});
this.getService = function(serviceName) {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'GET';
request_options.path = PATH_PREFIX + 'clusters/' + cluster.name + '/services/' + serviceName;
return performRequest(request_options).then(function(res) {
return new Service(cluster, res.body);
});
};
this.addService = function(serviceName) {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'POST';
request_options.path = PATH_PREFIX + 'clusters/' + cluster.name + '/services';
request_options.body = {
ServiceInfo: { service_name: serviceName }
};
return performRequest(request_options).then(function(res) {
return cluster.getService(serviceName);
});
};
this.removeService = function(service) {
if (!isString(service)) {
service = service.name;
}
var request_options = cluster.server.getBaseRequest();
request_options.method = 'DELETE';
request_options.path = PATH_PREFIX + 'clusters/' + cluster.name + '/services/' + service;
return performRequest(request_options);
};
this.getConfiguration = function() { throw new Error('Not implemented'); };
this.addConfiguration = function(type, tag, config) {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'POST';
request_options.path = PATH_PREFIX + 'clusters/' + cluster.name + '/configurations';
request_options.body = {
type: type,
tag: tag,
properties: config,
};
return performRequest(request_options);
};
this.removeConfiguration = function() { throw new Error('Not implemented'); };
this.updateConfiguration = function() { throw new Error('Not implemented'); };
this.getHost = function(hostname) {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'GET';
request_options.path = PATH_PREFIX + 'clusters/' + cluster.name + '/hosts/' + hostname;
return performRequest(request_options).then(function(res) {
return new Host(cluster, res.body);
});
};
this.addHost = function(hostname) {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'POST';
request_options.path = PATH_PREFIX + 'clusters/' + cluster.name + '/hosts';
request_options.body = { Hosts: { host_name: hostname} };
return performRequest(request_options).then(function(res) {
return cluster.getHost(hostname);
});
};
this.removeHost = function(hostname) {
var request_options = cluster.server.getBaseRequest();
request_options.method = 'DELETE';
request_options.path = PATH_PREFIX + 'clusters/' + cluster.name + '/hosts';
return performRequest(request_options);
};
this.updateHost = function() { throw new Error('Not implemented'); };
return this;
};
exports.Server = function(host, port, user, pass) {
var server = this;
this.host = host;
this.port = port;
this.user = user;
this.pass = pass;
this.getBaseRequest = function() {
return {
host: host,
port: port.toString(),
auth: user + ':' + pass
};
};
this.listClusterNames = function() {
var request = server.getBaseRequest();
request.path = PATH_PREFIX + 'clusters';
request.method = 'GET';
return performRequest(request).then(function(res) {
return res.body.items.map(function(clusterInfo) {
return clusterInfo.Clusters.cluster_name;
});
});
};
this.getCluster = function(clusterName) {
var request = server.getBaseRequest();
request.path = PATH_PREFIX + 'clusters/' + clusterName;
request.method = 'GET';
return performRequest(request).then(function(res) {
return new Cluster(server, res.body);
});
};
this.createCluster = function(clusterName, version) {
var request = server.getBaseRequest();
request.path = PATH_PREFIX + 'clusters/' + clusterName;
request.method = 'POST';
request.body = { "Clusters": { "version": version }};
return performRequest(request).then(function(res) {
return server.getCluster(clusterName);
});
};
this.updateCluster = function() { throw new Error('Not implemented'); };
this.removeCluster = function(clusterName) {
var request = server.getBaseRequest();
request.path = PATH_PREFIX + 'clusters/' + clusterName;
request.method = 'DELETE';
return performRequest(request);
};
return this;
};
;'use strict';
process.on('uncaughtException', function(err) {
console.log(err);
console.log(err.stack);
});
var Q = require('q');
var ambari = require('./ambari');
var server = new ambari.Server('localhost', 8080, 'admin', 'admin');
server.createCluster('test', 'HDP-1.2.0').then(function(cluster) {
var serviceD = Q.defer();
var servicePromise = serviceD.promise;
Q.all([
cluster.addService('HDFS'),
cluster.addConfiguration('core-site', '1', {
'fs.default.name' : 'local',
'fs.checkpoint.dir': '/hdfs/checkpoint',
'hadoop.security.authentication': 'simple',
'hadoop.security.authorization': 'false'
}),
cluster.addConfiguration('hdfs-site', '1', {
'dfs.name.dir': '/hdfs/name',
'dfs.support.append': 'true',
'dfs.webhdfs.enabled': 'false',
'dfs.datanode.failed.volume.tolerated': '0',
'dfs.block.local-path-access.user': 'root',
'dfs.data.dir': '/hdfs/data',
'dfs.replication': '1',
'dfs.datanode.address': '0.0.0.0:50010',
'dfs.datanode.http.address': '0.0.0.0:50075',
'dfs.http.address': '0.0.0.0:50070',
})
]).spread(function(service) {
serviceD.resolve(service);
});
return servicePromise.then(function(service) {
return Q.all([
service.addComponent('NAMENODE'),
service.addComponent('DATANODE'),
service.addComponent('HDFS_CLIENT')
]).then(function() { return service });
}).then(function(service) {
return Q.all([
service.cluster.addHost('1ddf42d1-0539-45f7-807a-a846a0cab9d8').then(function(host) {
return host.addComponents(['NAMENODE', 'DATANODE', 'HDFS_CLIENT']);
}),
service.applyConfiguration('core-site', '1'),
service.applyConfiguration('hdfs-site', '1')
]).then(function() { return service; });
}).then(function(service) {
return service.install().then(function() { return service; });
});
}).done();
