|
0
|
1 # $Id: HTTPget.pm,v 1.4 2002/10/22 07:38:37 lapp Exp $
|
|
|
2 #
|
|
|
3 # BioPerl module for fallback HTTP get operations.
|
|
|
4 # Module is proxy-aware
|
|
|
5 #
|
|
|
6 # Cared for by Chris Dagdigian <dag@sonsorol.org>
|
|
|
7 # but all of the good stuff was written by
|
|
|
8 # Lincoln Stein.
|
|
|
9 #
|
|
|
10 # You may distribute this module under the same terms as perl itself
|
|
|
11
|
|
|
12 # POD documentation - main docs before the code
|
|
|
13
|
|
|
14 =head1 NAME
|
|
|
15
|
|
|
16 Bio::Root::HTTPget - module for fallback HTTP get operations when
|
|
|
17 LWP:: is unavailable
|
|
|
18
|
|
|
19 =head1 SYNOPSIS
|
|
|
20
|
|
|
21 Use Bio::Root::HTTPget;
|
|
|
22
|
|
|
23 my $response = get('http://localhost');
|
|
|
24 $response = get('http://localhost/images');
|
|
|
25
|
|
|
26 $response = eval { get('http://fred:secret@localhost/ladies_only/')
|
|
|
27 } or warn $@;
|
|
|
28
|
|
|
29 $response = eval { get('http://jeff:secret@localhost/ladies_only/')
|
|
|
30 } or warn $@;
|
|
|
31
|
|
|
32 $response = get('http://localhost/images/navauthors.gif');
|
|
|
33 $response = get(-url=>'http://www.google.com',
|
|
|
34 -proxy=>'http://www.modperl.com');
|
|
|
35
|
|
|
36 =head1 DESCRIPTION
|
|
|
37
|
|
|
38 This is basically an last-chance module for doing network HTTP get requests in
|
|
|
39 situations where more advanced external CPAN modules such as LWP:: are not
|
|
|
40 installed.
|
|
|
41
|
|
|
42 The particular reason this module was developed was so that the Open Bio
|
|
|
43 Database Access code can fallback to fetching the default registry files
|
|
|
44 from http://open-bio.org/registry/ without having to depend on
|
|
|
45 external dependencies like Bundle::LWP for network HTTP access.
|
|
|
46
|
|
|
47 The core of this module was written by Lincoln Stein. It can handle proxies
|
|
|
48 and HTTP-based proxy authentication.
|
|
|
49
|
|
|
50 =head1 FEEDBACK
|
|
|
51
|
|
|
52 =head2 Mailing Lists
|
|
|
53
|
|
|
54 User feedback is an integral part of the evolution of this
|
|
|
55 and other Bioperl modules. Send your comments and suggestions preferably
|
|
|
56 to one of the Bioperl mailing lists.
|
|
|
57 Your participation is much appreciated.
|
|
|
58
|
|
|
59 bioperl-l@bioperl.org - General discussion
|
|
|
60 http://bio.perl.org/MailList.html - About the mailing lists
|
|
|
61
|
|
|
62 =head2 Reporting Bugs
|
|
|
63
|
|
|
64 Report bugs to the Bioperl bug tracking system to help us keep track
|
|
|
65 the bugs and their resolution.
|
|
|
66 Bug reports can be submitted via email or the web:
|
|
|
67
|
|
|
68 bioperl-bugs@bio.perl.org
|
|
|
69 http://bugzilla.bioperl.org/
|
|
|
70
|
|
|
71 =head1 AUTHOR - Lincoln Stein
|
|
|
72
|
|
|
73 Cared for by Chris Dagdigian <dag@sonsorol.org>
|
|
|
74
|
|
|
75 =head1 APPENDIX
|
|
|
76
|
|
|
77 The rest of the documentation details each of the object methods. Internal methods are usually preceded with a _
|
|
|
78
|
|
|
79 =cut
|
|
|
80
|
|
|
81
|
|
|
82 # Let the code begin...
|
|
|
83
|
|
|
84 package Bio::Root::HTTPget;
|
|
|
85
|
|
|
86 use strict;
|
|
|
87 use Bio::Root::Root;
|
|
|
88 use IO::Socket qw(:DEFAULT :crlf);
|
|
|
89 use vars '@ISA';
|
|
|
90
|
|
|
91 @ISA = qw(Bio::Root::Root);
|
|
|
92
|
|
|
93
|
|
|
94 =head2 get
|
|
|
95
|
|
|
96 Title : get
|
|
|
97 Usage :
|
|
|
98 Function:
|
|
|
99 Example :
|
|
|
100 Returns : string
|
|
|
101 Args :
|
|
|
102
|
|
|
103 =cut
|
|
|
104
|
|
|
105 sub get {
|
|
|
106 my ($url,$proxy,$timeout,$auth_user,$auth_pass) =
|
|
|
107 __PACKAGE__->_rearrange([qw(URL PROXY TIMEOUT USER PASS)],@_);
|
|
|
108 my $dest = $proxy || $url;
|
|
|
109
|
|
|
110 my ($host,$port,$path,$user,$pass)
|
|
|
111 = _http_parse_url($dest) or __PACKAGE__->throw("invalid URL $url");
|
|
|
112 $auth_user ||= $user;
|
|
|
113 $auth_pass ||= $pass;
|
|
|
114 $path = $url if $proxy;
|
|
|
115
|
|
|
116 # set up the connection
|
|
|
117 my $socket = _http_connect($host,$port) or __PACKAGE__->throw("can't connect: $@");
|
|
|
118
|
|
|
119 # the request
|
|
|
120 print $socket "GET $path HTTP/1.0$CRLF";
|
|
|
121 print $socket "User-Agent: Bioperl fallback fetcher/1.0$CRLF";
|
|
|
122 # Support virtual hosts
|
|
|
123 print $socket "HOST: $host$CRLF";
|
|
|
124
|
|
|
125 if ($auth_user && $auth_pass) { # authentication information
|
|
|
126 my $token = _encode_base64("$auth_user:$auth_pass");
|
|
|
127 print $socket "Authorization: Basic $token$CRLF";
|
|
|
128 }
|
|
|
129 print $socket "$CRLF";
|
|
|
130
|
|
|
131 # read the response
|
|
|
132 my $response;
|
|
|
133 {
|
|
|
134 local $/ = "$CRLF$CRLF";
|
|
|
135 $response = <$socket>;
|
|
|
136 }
|
|
|
137
|
|
|
138 my ($status_line,@other_lines) = split $CRLF,$response;
|
|
|
139 my ($stat_code,$stat_msg) = $status_line =~ m!^HTTP/1\.[01] (\d+) (.+)!
|
|
|
140 or __PACKAGE__->throw("invalid response from web server: got $response");
|
|
|
141
|
|
|
142 my %headers = map {/^(\S+): (.+)/} @other_lines;
|
|
|
143 if ($stat_code == 302 || $stat_code == 301) { # redirect
|
|
|
144 my $location = $headers{Location} or __PACKAGE__->throw("invalid redirect: no Location header");
|
|
|
145 return get($location,$proxy,$timeout); # recursive call
|
|
|
146 }
|
|
|
147
|
|
|
148 elsif ($stat_code == 401) { # auth required
|
|
|
149 my $auth_required = $headers{'WWW-Authenticate'};
|
|
|
150 $auth_required =~ /^Basic realm="([^\"]+)"/
|
|
|
151 or __PACKAGE__->throw("server requires unknown type of authentication: $auth_required");
|
|
|
152 __PACKAGE__->throw("request failed: $status_line, realm = $1");
|
|
|
153 }
|
|
|
154
|
|
|
155 elsif ($stat_code != 200) {
|
|
|
156 __PACKAGE__->throw("request failed: $status_line");
|
|
|
157 }
|
|
|
158
|
|
|
159 $response = '';
|
|
|
160 while (1) {
|
|
|
161 my $bytes = read($socket,$response,2048,length $response);
|
|
|
162 last unless $bytes > 0;
|
|
|
163 }
|
|
|
164
|
|
|
165 $response;
|
|
|
166 }
|
|
|
167
|
|
|
168 =head2 getFH
|
|
|
169
|
|
|
170 Title : getFH
|
|
|
171 Usage :
|
|
|
172 Function:
|
|
|
173 Example :
|
|
|
174 Returns : string
|
|
|
175 Args :
|
|
|
176
|
|
|
177 =cut
|
|
|
178
|
|
|
179 sub getFH {
|
|
|
180 my ($url,$proxy,$timeout,$auth_user,$auth_pass) =
|
|
|
181 __PACKAGE__->_rearrange([qw(URL PROXY TIMEOUT USER PASS)],@_);
|
|
|
182 my $dest = $proxy || $url;
|
|
|
183
|
|
|
184 my ($host,$port,$path,$user,$pass)
|
|
|
185 = _http_parse_url($dest) or __PACKAGE__->throw("invalid URL $url");
|
|
|
186 $auth_user ||= $user;
|
|
|
187 $auth_pass ||= $pass;
|
|
|
188 $path = $url if $proxy;
|
|
|
189
|
|
|
190 # set up the connection
|
|
|
191 my $socket = _http_connect($host,$port) or __PACKAGE__->throw("can't connect: $@");
|
|
|
192
|
|
|
193 # the request
|
|
|
194 print $socket "GET $path HTTP/1.0$CRLF";
|
|
|
195 print $socket "User-Agent: Bioperl fallback fetcher/1.0$CRLF";
|
|
|
196 # Support virtual hosts
|
|
|
197 print $socket "HOST: $host$CRLF";
|
|
|
198
|
|
|
199 if ($auth_user && $auth_pass) { # authentication information
|
|
|
200 my $token = _encode_base64("$auth_user:$auth_pass");
|
|
|
201 print $socket "Authorization: Basic $token$CRLF";
|
|
|
202 }
|
|
|
203 print $socket "$CRLF";
|
|
|
204
|
|
|
205 # read the response
|
|
|
206 my $response;
|
|
|
207 {
|
|
|
208 local $/ = "$CRLF$CRLF";
|
|
|
209 $response = <$socket>;
|
|
|
210 }
|
|
|
211
|
|
|
212 my ($status_line,@other_lines) = split $CRLF,$response;
|
|
|
213 my ($stat_code,$stat_msg) = $status_line =~ m!^HTTP/1\.[01] (\d+) (.+)!
|
|
|
214 or __PACKAGE__->throw("invalid response from web server: got $response");
|
|
|
215
|
|
|
216 my %headers = map {/^(\S+): (.+)/} @other_lines;
|
|
|
217 if ($stat_code == 302 || $stat_code == 301) { # redirect
|
|
|
218 my $location = $headers{Location} or __PACKAGE__->throw("invalid redirect: no Location header");
|
|
|
219 return get($location,$proxy,$timeout); # recursive call
|
|
|
220 }
|
|
|
221
|
|
|
222 elsif ($stat_code == 401) { # auth required
|
|
|
223 my $auth_required = $headers{'WWW-Authenticate'};
|
|
|
224 $auth_required =~ /^Basic realm="([^\"]+)"/
|
|
|
225 or __PACKAGE__->throw("server requires unknown type of authentication: $auth_required");
|
|
|
226 __PACKAGE__->throw("request failed: $status_line, realm = $1");
|
|
|
227 }
|
|
|
228
|
|
|
229 elsif ($stat_code != 200) {
|
|
|
230 __PACKAGE__->throw("request failed: $status_line");
|
|
|
231 }
|
|
|
232
|
|
|
233 # Now that we are reasonably sure the socket and request
|
|
|
234 # are OK we pass the socket back as a filehandle so it can
|
|
|
235 # be processed by the caller...
|
|
|
236
|
|
|
237 $socket;
|
|
|
238
|
|
|
239 }
|
|
|
240
|
|
|
241
|
|
|
242 =head2 _http_parse_url
|
|
|
243
|
|
|
244 Title :
|
|
|
245 Usage :
|
|
|
246 Function:
|
|
|
247 Example :
|
|
|
248 Returns :
|
|
|
249 Args :
|
|
|
250
|
|
|
251 =cut
|
|
|
252
|
|
|
253 sub _http_parse_url {
|
|
|
254 my $url = shift;
|
|
|
255 my ($user,$pass,$hostent,$path) =
|
|
|
256 $url =~ m!^http://(?:([^:]+):([^:]+)@)?([^/]+)(/?[^\#]*)! or return;
|
|
|
257 $path ||= '/';
|
|
|
258 my ($host,$port) = split(':',$hostent);
|
|
|
259 return ($host,$port||80,$path,$user,$pass);
|
|
|
260 }
|
|
|
261
|
|
|
262 =head2 _http_connect
|
|
|
263
|
|
|
264 Title :
|
|
|
265 Usage :
|
|
|
266 Function:
|
|
|
267 Example :
|
|
|
268 Returns :
|
|
|
269 Args :
|
|
|
270
|
|
|
271 =cut
|
|
|
272
|
|
|
273 sub _http_connect {
|
|
|
274 my ($host,$port,$timeout) = @_;
|
|
|
275 my $sock = IO::Socket::INET->new(Proto => 'tcp',
|
|
|
276 Type => SOCK_STREAM,
|
|
|
277 PeerHost => $host,
|
|
|
278 PeerPort => $port,
|
|
|
279 Timeout => $timeout,
|
|
|
280 );
|
|
|
281 $sock;
|
|
|
282 }
|
|
|
283
|
|
|
284
|
|
|
285 =head2 _encode_base64
|
|
|
286
|
|
|
287 Title :
|
|
|
288 Usage :
|
|
|
289 Function:
|
|
|
290 Example :
|
|
|
291 Returns :
|
|
|
292 Args :
|
|
|
293
|
|
|
294 =cut
|
|
|
295
|
|
|
296 sub _encode_base64 {
|
|
|
297 my $res = "";
|
|
|
298 my $eol = $_[1];
|
|
|
299 $eol = "\n" unless defined $eol;
|
|
|
300 pos($_[0]) = 0; # ensure start at the beginning
|
|
|
301
|
|
|
302 $res = join '', map( pack('u',$_)=~ /^.(\S*)/, ($_[0]=~/(.{1,45})/gs));
|
|
|
303
|
|
|
304 $res =~ tr|` -_|AA-Za-z0-9+/|; # `# help emacs
|
|
|
305 # fix padding at the end
|
|
|
306 my $padding = (3 - length($_[0]) % 3) % 3;
|
|
|
307 $res =~ s/.{$padding}$/'=' x $padding/e if $padding;
|
|
|
308 # break encoded string into lines of no more than 76 characters each
|
|
|
309 if (length $eol) {
|
|
|
310 $res =~ s/(.{1,76})/$1$eol/g;
|
|
|
311 }
|
|
|
312 return $res;
|
|
|
313 }
|
|
|
314
|
|
|
315 1;
|
