Author: assaf
Date: Mon May 19 20:31:38 2008
New Revision: 658093
URL: http://svn.apache.org/viewvc?rev=658093&view=rev
Log:
Added HTTP upload for remote repositories that support it (proxy, basic auth,
SSL, all other good things).
Modified:
incubator/buildr/trunk/CHANGELOG
incubator/buildr/trunk/lib/buildr/core/transports.rb
incubator/buildr/trunk/spec/transport_spec.rb
Modified: incubator/buildr/trunk/CHANGELOG
URL:
http://svn.apache.org/viewvc/incubator/buildr/trunk/CHANGELOG?rev=658093&r1=658092&r2=658093&view=diff
==============================================================================
--- incubator/buildr/trunk/CHANGELOG (original)
+++ incubator/buildr/trunk/CHANGELOG Mon May 19 20:31:38 2008
@@ -1,6 +1,6 @@
-1.3.1 (Pending)
-* Added: Reading files from SFTP server.
-* Added: jibx_bind method to use JiBX or Java<=>XML binding (by David
+1.3.1 (2008-05-15)
+* Added: Downloading files from SFTP server, uploading to HTTP.
+* Added: jibx_bind method to use JiBX for Java<=>XML binding (by David
Peterson).
* Changed: Upgraded to Net::SSH 2.0 and Net::SFTP 2.0.
* Fixed: BUILDR-67 HTTP GET now works with query parameters (Tommy Knowlton).
Modified: incubator/buildr/trunk/lib/buildr/core/transports.rb
URL:
http://svn.apache.org/viewvc/incubator/buildr/trunk/lib/buildr/core/transports.rb?rev=658093&r1=658092&r2=658093&view=diff
==============================================================================
--- incubator/buildr/trunk/lib/buildr/core/transports.rb (original)
+++ incubator/buildr/trunk/lib/buildr/core/transports.rb Mon May 19 20:31:38
2008
@@ -22,6 +22,7 @@
require 'uri'
require 'digest/md5'
require 'digest/sha1'
+require 'stringio'
require 'tempfile'
require 'buildr/core/progressbar'
@@ -264,11 +265,11 @@
end
# :call-seq:
- # proxy_uri() => URI?
+ # proxy_uri => URI?
#
# Returns the proxy server to use. Obtains the proxy from the relevant
environment variable (e.g. HTTP_PROXY).
# Supports exclusions based on host name and port number from environment
variable NO_PROXY.
- def proxy_uri()
+ def proxy_uri
proxy = ENV["#{scheme.upcase}_PROXY"]
proxy = URI.parse(proxy) if String === proxy
excludes = ENV['NO_PROXY'].to_s.split(/\s*,\s*/).compact
@@ -288,56 +289,99 @@
# See URI::Generic#read
def read(options = nil, &block)
options ||= {}
- headers = { 'If-Modified-Since' =>
CGI.rfc1123_date(options[:modified].utc) } if options[:modified]
-
- if proxy = proxy_uri
- proxy = URI.parse(proxy) if String === proxy
- http = Net::HTTP.new(host, port, proxy.host, proxy.port, proxy.user,
proxy.password)
- else
- http = Net::HTTP.new(host, port)
+ connect do |http|
+ puts "Requesting #{self}" if Buildr.application.options.trace
+ headers = { 'If-Modified-Since' =>
CGI.rfc1123_date(options[:modified].utc) } if options[:modified]
+ request = Net::HTTP::Get.new(request_uri.empty? ? '/' : request_uri,
headers)
+ request.basic_auth self.user, self.password if self.user
+ http.request request do |response|
+ case response
+ when Net::HTTPNotModified
+ # No modification, nothing to do.
+ puts 'Not modified since last download' if
Buildr.application.options.trace
+ return nil
+ when Net::HTTPRedirection
+ # Try to download from the new URI, handle relative redirects.
+ puts "Redirected to #{response['Location']}" if
Buildr.application.options.trace
+ return (self + URI.parse(response['location'])).read(options,
&block)
+ when Net::HTTPOK
+ puts "Downloading #{self}" if verbose
+ result = nil
+ with_progress_bar options[:progress], path.split('/').last,
response.content_length do |progress|
+ if block
+ response.read_body do |chunk|
+ block.call chunk
+ progress << chunk
+ end
+ else
+ result = ''
+ response.read_body do |chunk|
+ result << chunk
+ progress << chunk
+ end
+ end
+ end
+ return result
+ when Net::HTTPNotFound
+ raise NotFoundError, "Looking for #{self} and all I got was a 404!"
+ else
+ raise RuntimeError, "Failed to download #{self}:
#{response.message}"
+ end
+ end
end
- http.use_ssl = true if self.instance_of? URI::HTTPS
+ end
+
+ private
+
+ def write_internal(options, &block) #:nodoc:
+ options ||= {}
+ connect do |http|
+ puts "Uploading to #{path}" if Buildr.application.options.trace
+ content = StringIO.new
+ while chunk = yield(32 * 4096)
+ content << chunk
+ end
+ headers = { 'Content-MD5'=>Digest::MD5.hexdigest(content.string) }
+ request = Net::HTTP::Put.new(request_uri.empty? ? '/' : request_uri,
headers)
+ request.basic_auth self.user, self.password if self.user
+ response = nil
+ with_progress_bar options[:progress], path.split('/').last,
content.size do |progress|
+ request.content_length = content.size
+ content.rewind
+ stream = Object.new
+ class << stream ; self ;end.send :define_method, :read do |count|
+ bytes = content.read(count)
+ progress << bytes if bytes
+ bytes
+ end
+ request.body_stream = stream
+ response = http.request(request)
+ end
- puts "Requesting #{self}" if Buildr.application.options.trace
- request = Net::HTTP::Get.new(request_uri.empty? ? '/' : request_uri,
headers)
- request.basic_auth self.user, self.password if self.user
- http.request request do |response|
case response
- #case response = http.request(request)
- when Net::HTTPNotModified
- # No modification, nothing to do.
- puts 'Not modified since last download' if
Buildr.application.options.trace
- return nil
when Net::HTTPRedirection
# Try to download from the new URI, handle relative redirects.
puts "Redirected to #{response['Location']}" if
Buildr.application.options.trace
- return (self + URI.parse(response['location'])).read(options, &block)
- when Net::HTTPOK
- puts "Downloading #{self}" if verbose
- result = nil
- with_progress_bar options[:progress], path.split('/').last,
response.content_length do |progress|
- if block
- response.read_body do |chunk|
- block.call chunk
- progress << chunk
- end
- else
- result = ''
- response.read_body do |chunk|
- result << chunk
- progress << chunk
- end
- end
- end
- return result
- when Net::HTTPNotFound
- raise NotFoundError, "Looking for #{self} and all I got was a 404!"
+ content.rewind
+ return (self +
URI.parse(response['location'])).write_internal(options) { |bytes|
content.read(bytes) }
+ when Net::HTTPSuccess
else
- raise RuntimeError, "Failed to download #{self}: #{response.message}"
+ raise RuntimeError, "Failed to upload #{self}: #{response.message}"
end
end
end
+ def connect
+ if proxy = proxy_uri
+ proxy = URI.parse(proxy) if String === proxy
+ http = Net::HTTP.new(host, port, proxy.host, proxy.port, proxy.user,
proxy.password)
+ else
+ http = Net::HTTP.new(host, port)
+ end
+ http.use_ssl = true if self.instance_of? URI::HTTPS
+ yield http
+ end
+
end
@@ -490,14 +534,14 @@
end
end
- def to_s()
+ def to_s
"file://#{host}#{path}"
end
# The URL path always starts with a backslash. On most operating systems
(Linux, Darwin, BSD) it points
# to the absolute path on the file system. But on Windows, it comes before
the drive letter, creating an
# unusable path, so real_path fixes that. Ugly but necessary hack.
- def real_path() #:nodoc:
+ def real_path #:nodoc:
RUBY_PLATFORM =~ /win32/ && path =~ /^\/[a-zA-Z]:\// ? path[1..-1] : path
end
Modified: incubator/buildr/trunk/spec/transport_spec.rb
URL:
http://svn.apache.org/viewvc/incubator/buildr/trunk/spec/transport_spec.rb?rev=658093&r1=658092&r2=658093&view=diff
==============================================================================
--- incubator/buildr/trunk/spec/transport_spec.rb (original)
+++ incubator/buildr/trunk/spec/transport_spec.rb Mon May 19 20:31:38 2008
@@ -311,57 +311,81 @@
describe URI::HTTP, '#write' do
-end
-
-
-describe URI::SFTP, '#write' do
before do
- @uri = URI('sftp://john:[EMAIL PROTECTED]/path/readme')
@content = 'Readme. Please!'
+ @uri = URI('http://john:[EMAIL PROTECTED]/foo/bar/baz.jar')
+ @http = mock('Net::HTTP')
+ @http.stub!(:request).and_return(Net::HTTPOK.new(nil, nil, nil))
+ Net::HTTP.stub!(:new).and_return(@http)
+ end
- @ssh_session = mock('Net::SSH::Session')
- @sftp_session = mock('Net::SFTP::Session')
- @file_factory = mock('Net::SFTP::Operations::FileFactory')
- Net::SSH.stub!(:start).with('localhost', 'john', :password=>'secret',
:port=>22) do
-
Net::SFTP::Session.should_receive(:new).with(@ssh_session).and_yield(@sftp_session).and_return(@sftp_session)
- @sftp_session.should_receive(:connect!).and_return(@sftp_session)
- @sftp_session.should_receive(:loop)
- @sftp_session.stub!(:mkdir)
- @sftp_session.should_receive(:file).with.and_return(@file_factory)
- @file_factory.stub!(:open)
- @ssh_session.should_receive(:close)
- @ssh_session
- end
+ it 'should open connection to HTTP server' do
+ Net::HTTP.should_receive(:new).with('host.domain', 80).and_return(@http)
+ @uri.write @content
end
- it 'should open connection to SFTP server' do
+ it 'should use HTTP basic authentication' do
+ @http.should_receive(:request) do |request|
+ request['authorization'].should == ('Basic ' +
['john:secret'].pack('m').delete("\r\n"))
+ Net::HTTPOK.new(nil, nil, nil)
+ end
@uri.write @content
end
- it 'should create path recursively on SFTP server' do
- @sftp_session.should_receive(:mkdir).ordered.with('', {})
- @sftp_session.should_receive(:mkdir).ordered.with('/path', {})
+ it 'should use HTTPS if applicable' do
+ Net::HTTP.should_receive(:new).with('host.domain', 443).and_return(@http)
+ @http.should_receive(:use_ssl=).with(true)
+ URI(@uri.to_s.sub(/http/, 'https')).write @content
+ end
+
+ it 'should upload file with PUT request' do
+ @http.should_receive(:request) do |request|
+ request.should be_kind_of(Net::HTTP::Put)
+ Net::HTTPOK.new(nil, nil, nil)
+ end
@uri.write @content
end
- it 'should only create paths that don\'t exist' do
- @sftp_session.should_receive(:realpath).any_number_of_times
- @sftp_session.should_not_receive(:mkdir)
+ it 'should set Content-Length header' do
+ @http.should_receive(:request) do |request|
+ request.content_length.should == @content.size
+ Net::HTTPOK.new(nil, nil, nil)
+ end
@uri.write @content
end
- it 'should open file for writing' do
- @file_factory.should_receive(:open).with('/path/readme', 'w')
+ it 'should set Content-MD5 header' do
+ @http.should_receive(:request) do |request|
+ request['Content-MD5'].should == Digest::MD5.hexdigest(@content)
+ Net::HTTPOK.new(nil, nil, nil)
+ end
@uri.write @content
end
- it 'should write contents to file' do
- file = mock('Net::SFTP::Operations::File')
- file.should_receive(:write).with(@content)
- @file_factory.should_receive(:open).with('/path/readme',
'w').and_yield(file)
+ it 'should send entire content' do
+ @http.should_receive(:request) do |request|
+ body_stream = request.body_stream
+ body_stream.read(1024).should == @content
+ body_stream.read(1024).should be_nil
+ Net::HTTPOK.new(nil, nil, nil)
+ end
@uri.write @content
end
+ it 'should fail on 4xx response' do
+ @http.should_receive(:request) do |request|
+ Net::HTTPBadRequest.new(nil, nil, nil)
+ end
+ lambda { @uri.write @content }.should raise_error(RuntimeError, /failed to
upload/i)
+ end
+
+ it 'should fail on 5xx response' do
+ @http.should_receive(:request) do |request|
+ Net::HTTPServiceUnavailable.new(nil, nil, nil)
+ end
+ lambda { @uri.write @content }.should raise_error(RuntimeError, /failed to
upload/i)
+ end
+
end
@@ -411,3 +435,54 @@
content.should eql(@content)
end
end
+
+
+describe URI::SFTP, '#write' do
+ before do
+ @uri = URI('sftp://john:[EMAIL PROTECTED]/path/readme')
+ @content = 'Readme. Please!'
+
+ @ssh_session = mock('Net::SSH::Session')
+ @sftp_session = mock('Net::SFTP::Session')
+ @file_factory = mock('Net::SFTP::Operations::FileFactory')
+ Net::SSH.stub!(:start).with('localhost', 'john', :password=>'secret',
:port=>22) do
+
Net::SFTP::Session.should_receive(:new).with(@ssh_session).and_yield(@sftp_session).and_return(@sftp_session)
+ @sftp_session.should_receive(:connect!).and_return(@sftp_session)
+ @sftp_session.should_receive(:loop)
+ @sftp_session.stub!(:mkdir)
+ @sftp_session.should_receive(:file).with.and_return(@file_factory)
+ @file_factory.stub!(:open)
+ @ssh_session.should_receive(:close)
+ @ssh_session
+ end
+ end
+
+ it 'should open connection to SFTP server' do
+ @uri.write @content
+ end
+
+ it 'should create path recursively on SFTP server' do
+ @sftp_session.should_receive(:mkdir).ordered.with('', {})
+ @sftp_session.should_receive(:mkdir).ordered.with('/path', {})
+ @uri.write @content
+ end
+
+ it 'should only create paths that don\'t exist' do
+ @sftp_session.should_receive(:realpath).any_number_of_times
+ @sftp_session.should_not_receive(:mkdir)
+ @uri.write @content
+ end
+
+ it 'should open file for writing' do
+ @file_factory.should_receive(:open).with('/path/readme', 'w')
+ @uri.write @content
+ end
+
+ it 'should write contents to file' do
+ file = mock('Net::SFTP::Operations::File')
+ file.should_receive(:write).with(@content)
+ @file_factory.should_receive(:open).with('/path/readme',
'w').and_yield(file)
+ @uri.write @content
+ end
+
+end
