ExtensionsLst.pm (c6dedb65) ExtensionsLst.pm (201b78c7)
1#**************************************************************
2#
3# Licensed to the Apache Software Foundation (ASF) under one
4# or more contributor license agreements. See the NOTICE file
5# distributed with this work for additional information
6# regarding copyright ownership. The ASF licenses this file
7# to you under the Apache License, Version 2.0 (the
8# "License"); you may not use this file except in compliance

--- 266 unchanged lines hidden (view full) ---

275 }
276}
277
278
279
280
281=head3 ProcessURL
282 Check that the given line contains an optional MD5 sum followed by
1#**************************************************************
2#
3# Licensed to the Apache Software Foundation (ASF) under one
4# or more contributor license agreements. See the NOTICE file
5# distributed with this work for additional information
6# regarding copyright ownership. The ASF licenses this file
7# to you under the Apache License, Version 2.0 (the
8# "License"); you may not use this file except in compliance

--- 266 unchanged lines hidden (view full) ---

275 }
276}
277
278
279
280
281=head3 ProcessURL
282 Check that the given line contains an optional MD5 sum followed by
283 a URL for one of the protocols file, http, https
283 a URL for one of the protocols file, http, https,
284 followed by an optional file name (which is necessary when it is not the last part of the URL.)
284 Return an array that contains the protocol, the name, the original
285 URL, and the MD5 sum from the beginning of the line.
286 The name of the URL depends on its protocol:
287 - for http(s) the part of the URL after the last '/'.
288 - for file URLS it is everything after the protocol://
289=cut
290sub ProcessURL ($)
291{
292 my $line = shift;
293
294 # Check that we are looking at a valid URL.
285 Return an array that contains the protocol, the name, the original
286 URL, and the MD5 sum from the beginning of the line.
287 The name of the URL depends on its protocol:
288 - for http(s) the part of the URL after the last '/'.
289 - for file URLS it is everything after the protocol://
290=cut
291sub ProcessURL ($)
292{
293 my $line = shift;
294
295 # Check that we are looking at a valid URL.
295 if ($line =~ /^\s*(\w{32}\s+)?([a-zA-Z]+)(:\/\/.*?\/)([^\/ \t]+)\s*$/)
296 if ($line =~ /^\s*((\w{32})\s+)?([a-zA-Z]+)(:\/\/.*?\/)([^\/ \t]+)(\s+\"[^\"]+\")?\s*$/)
296 {
297 {
297 my ($md5, $protocol, $name) = ($1,$2,$4);
298 my $URL = $2.$3.$4;
298 my ($md5, $protocol, $url_name, $optional_name) = ($2,$3,$5,$6);
299 my $URL = $3.$4.$5;
299
300 die "invalid URL protocol on line $LineNo:\n$line\n" if $protocol !~ /(file|http|https)/;
301
300
301 die "invalid URL protocol on line $LineNo:\n$line\n" if $protocol !~ /(file|http|https)/;
302
302 # For file URLs we use everything after :// as name.
303 if ($protocol eq "file")
303 # Determine the name. If an optional name is given then use that.
304 if (defined $optional_name)
304 {
305 {
305 $URL =~ /:\/\/(.*)$/;
306 die if $optional_name !~ /^\s+\"([^\"]+)\"$/;
306 $name = $1;
307 }
307 $name = $1;
308 }
309 else
310 {
311 if ($protocol eq "file")
312 {
313 # For file URLs we use everything after :// as name, or the .
314 $URL =~ /:\/\/(.*)$/;
315 $name = $1;
316 }
317 else
318 {
319 # For http and https use the last part of the URL.
320 $name = $url_name;
321 }
322 }
308
309 return [$protocol, $name, $URL, $md5];
310 }
311 else
312 {
313 die "invalid URL at line $LineNo:\n$line\n";
314 }
315}

--- 109 unchanged lines hidden (view full) ---

425 # Download the missing files.
426 for my $entry (@missing)
427 {
428 my ($protocol, $name, $URL, $md5sum) = @{$entry};
429
430 # Open a .part file for writing.
431 my $filename = File::Spec->catfile($download_path, $name);
432 my $temporary_filename = $filename . ".part";
323
324 return [$protocol, $name, $URL, $md5];
325 }
326 else
327 {
328 die "invalid URL at line $LineNo:\n$line\n";
329 }
330}

--- 109 unchanged lines hidden (view full) ---

440 # Download the missing files.
441 for my $entry (@missing)
442 {
443 my ($protocol, $name, $URL, $md5sum) = @{$entry};
444
445 # Open a .part file for writing.
446 my $filename = File::Spec->catfile($download_path, $name);
447 my $temporary_filename = $filename . ".part";
448 print "downloading to $temporary_filename\n";
433 open my $out, ">$temporary_filename";
434 binmode($out);
435
436 # Prepare md5
437 my $md5 = Digest::MD5->new();
438
439 # Download the extension.
440 my $agent = LWP::UserAgent->new();
441 $agent->timeout(10);
442 $agent->show_progress(1);
449 open my $out, ">$temporary_filename";
450 binmode($out);
451
452 # Prepare md5
453 my $md5 = Digest::MD5->new();
454
455 # Download the extension.
456 my $agent = LWP::UserAgent->new();
457 $agent->timeout(10);
458 $agent->show_progress(1);
459 my $last_was_redirect = 0;
460 $agent->add_handler('response_redirect'
461 => sub{
462 $last_was_redirect = 1;
463 return;
464 });
443 $agent->add_handler('response_data'
444 => sub{
465 $agent->add_handler('response_data'
466 => sub{
467 if ($last_was_redirect)
468 {
469 $last_was_redirect = 0;
470 # Throw away the data we got so far.
471 $md5->reset();
472 close $out;
473 open $out, ">$temporary_filename";
474 binmode($out);
475 }
445 my($response,$agent,$h,$data)=@_;
446 print $out $data;
447 $md5->add($data);
448 });
449 my $response = $agent->get($URL);
450 close $out;
451
452 # When download was successfull then check the md5 checksum and rename the .part file
453 # into the actual extension name.
454 if ($response->is_success())
455 {
456 if (defined $md5sum && length($md5sum)==32)
457 {
476 my($response,$agent,$h,$data)=@_;
477 print $out $data;
478 $md5->add($data);
479 });
480 my $response = $agent->get($URL);
481 close $out;
482
483 # When download was successfull then check the md5 checksum and rename the .part file
484 # into the actual extension name.
485 if ($response->is_success())
486 {
487 if (defined $md5sum && length($md5sum)==32)
488 {
458 if ($md5sum eq $md5->digest())
489 my $file_md5 = $md5->hexdigest();
490 if ($md5sum eq $file_md5)
459 {
460 print "md5 is OK\n";
461 }
462 else
463 {
464 unlink($temporary_filename);
491 {
492 print "md5 is OK\n";
493 }
494 else
495 {
496 unlink($temporary_filename);
465 die "downloaded file has the wrong md5 checksum";
497 die "downloaded file has the wrong md5 checksum: $file_md5 instead of $md5sum";
466 }
467 }
498 }
499 }
500 else
501 {
502 print "md5 is not present\n";
503 printf " is %s, length is %d\n", $md5sum, length(md5sum);
504 }
468
469 rename($temporary_filename, $filename) || die "can not rename $temporary_filename to $filename";
470 }
471 else
472 {
473 die "failed to download $URL";
474 }
475 }

--- 56 unchanged lines hidden --- 		505
506 rename($temporary_filename, $filename) || die "can not rename $temporary_filename to $filename";
507 }
508 else
509 {
510 die "failed to download $URL";
511 }
512 }

--- 56 unchanged lines hidden ---