1*c667dd47SPedro Giffuni#!/usr/bin/env perl 237e6b05aSAndre Fischer 396dbeebcSAndre Fischer#************************************************************** 496dbeebcSAndre Fischer# 596dbeebcSAndre Fischer# Licensed to the Apache Software Foundation (ASF) under one 696dbeebcSAndre Fischer# or more contributor license agreements. See the NOTICE file 796dbeebcSAndre Fischer# distributed with this work for additional information 896dbeebcSAndre Fischer# regarding copyright ownership. The ASF licenses this file 996dbeebcSAndre Fischer# to you under the Apache License, Version 2.0 (the 1096dbeebcSAndre Fischer# "License"); you may not use this file except in compliance 1196dbeebcSAndre Fischer# with the License. You may obtain a copy of the License at 1296dbeebcSAndre Fischer# 1396dbeebcSAndre Fischer# http://www.apache.org/licenses/LICENSE-2.0 1496dbeebcSAndre Fischer# 1596dbeebcSAndre Fischer# Unless required by applicable law or agreed to in writing, 1696dbeebcSAndre Fischer# software distributed under the License is distributed on an 1796dbeebcSAndre Fischer# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 1896dbeebcSAndre Fischer# KIND, either express or implied. See the License for the 1996dbeebcSAndre Fischer# specific language governing permissions and limitations 2096dbeebcSAndre Fischer# under the License. 2196dbeebcSAndre Fischer# 2296dbeebcSAndre Fischer#************************************************************** 2396dbeebcSAndre Fischer 2437e6b05aSAndre Fischer=head1 NAME 2537e6b05aSAndre Fischer 2637e6b05aSAndre Fischer download_external_libraries.pl - Load missing tarballs specified in main/external_libs.lst. 2737e6b05aSAndre Fischer 2837e6b05aSAndre Fischer=head1 SYNOPSIS 2937e6b05aSAndre Fischer 3037e6b05aSAndre Fischer For downloading external libraries (typically from the main/bootstrap script): 316a22bca6SAndre Fischer 3237e6b05aSAndre Fischer download_external_libraries(<data-file-name>); 3337e6b05aSAndre Fischer 3437e6b05aSAndre Fischer=head1 DESCRIPTION 3537e6b05aSAndre Fischer 3637e6b05aSAndre Fischer The contents of the main/external_libs.lst file are used to determine the 3737e6b05aSAndre Fischer external library tarballs that are missing from ext_sources/. 3837e6b05aSAndre Fischer 3937e6b05aSAndre Fischer Individual libraries can be ignored depending on the values of environment variables. 4037e6b05aSAndre Fischer 4137e6b05aSAndre Fischer Format of the main/external_libs.lst file: 4237e6b05aSAndre Fischer 4337e6b05aSAndre Fischer The file is line based. 4437e6b05aSAndre Fischer Comments start with a # and go to the end of the line and are ignored. 4537e6b05aSAndre Fischer Lines that are empty or contain only spaces and/or comments are ignored. 4637e6b05aSAndre Fischer 4737e6b05aSAndre Fischer All other lines can have one of two forms: 4837e6b05aSAndre Fischer - A variable definition of the form <name>=<value>. 4937e6b05aSAndre Fischer - A conditional block start in the form "if (<expression>)" 5037e6b05aSAndre Fischer 5137e6b05aSAndre Fischer Variables defined in a conditional block are only visible in this block and 5237e6b05aSAndre Fischer replace the definition of global variables and variables earlier in the same 5337e6b05aSAndre Fischer block. 5437e6b05aSAndre Fischer Some variables have special names: 5510e20387SAndre Fischer - MD5 is the expected MD5 checksum of the library tarball. 5610e20387SAndre Fischer - SHA1 is the expected SHA1 checksum of the library tarball. 5737e6b05aSAndre Fischer - URL1 to URL9 specify from where to download the tarball. The urls are tried in order. 5810e20387SAndre Fischer The first successful download (download completed and checksum match) stops the iteration. 5937e6b05aSAndre Fischer 6037e6b05aSAndre Fischer Expressions are explained below in the comment of EvaluateExpression(). 6137e6b05aSAndre Fischer 6237e6b05aSAndre Fischer A library is only regarded if its conditional expression evaluates to 1. 6337e6b05aSAndre Fischer 6437e6b05aSAndre Fischer Example: 6537e6b05aSAndre Fischer 6637e6b05aSAndre Fischer DefaultSite=http://some-internet-site.org 6737e6b05aSAndre Fischer if ( true ) 6837e6b05aSAndre Fischer MD5 = 0123456789abcdef0123456789abcdef 6937e6b05aSAndre Fischer name = library-1.0.tar.gz 7037e6b05aSAndre Fischer URL1 = http://some-other-internet-site.org/another-name.tgz 7137e6b05aSAndre Fischer URL2 = $(DefaultSite)$(MD5)-$(name) 7237e6b05aSAndre Fischer 7337e6b05aSAndre Fischer This tries to load a library first from some-other-internet-site.org and if 7437e6b05aSAndre Fischer that fails from some-internet-site.org. The library is stored as $(MD5)-$(name) 7537e6b05aSAndre Fischer even when it is loaded as another-name.tgz. 7637e6b05aSAndre Fischer 7737e6b05aSAndre Fischer=cut 7837e6b05aSAndre Fischer 7937e6b05aSAndre Fischer 8037e6b05aSAndre Fischeruse strict; 8137e6b05aSAndre Fischer 8237e6b05aSAndre Fischeruse File::Spec; 8337e6b05aSAndre Fischeruse File::Path; 8437e6b05aSAndre Fischeruse File::Basename; 8537e6b05aSAndre Fischeruse Digest::MD5; 8610e20387SAndre Fischeruse Digest::SHA; 8737e6b05aSAndre Fischeruse URI; 88f253223cSAndre Fischermy $simple = 1; 89f253223cSAndre Fischerif ($simple) 90f253223cSAndre Fischer{ 91f253223cSAndre Fischer use LWP::Simple; 92f253223cSAndre Fischer} 93f253223cSAndre Fischerelse 94f253223cSAndre Fischer{ 95f253223cSAndre Fischer use LWP::UserAgent; 96f253223cSAndre Fischer} 9737e6b05aSAndre Fischer 9837e6b05aSAndre Fischermy $Debug = 1; 9937e6b05aSAndre Fischer 10037e6b05aSAndre Fischermy $LocalEnvironment = undef; 10137e6b05aSAndre Fischermy $GlobalEnvironment = {}; 10237e6b05aSAndre Fischermy @Missing = (); 10337e6b05aSAndre Fischer 10437e6b05aSAndre Fischer 10537e6b05aSAndre Fischer 10637e6b05aSAndre Fischer 10737e6b05aSAndre Fischer=head3 ProcessDataFile 1086a22bca6SAndre Fischer 10937e6b05aSAndre Fischer Read the data file, typically named main/external_libs.lst, find the external 11037e6b05aSAndre Fischer library tarballs that are not yet present in ext_sources/ and download them. 11137e6b05aSAndre Fischer 11237e6b05aSAndre Fischer=cut 11337e6b05aSAndre Fischersub ProcessDataFile ($) 11437e6b05aSAndre Fischer{ 11537e6b05aSAndre Fischer my $filename = shift; 11637e6b05aSAndre Fischer 11737e6b05aSAndre Fischer my $destination = $ENV{'TARFILE_LOCATION'}; 11837e6b05aSAndre Fischer 11937e6b05aSAndre Fischer die "can not open data file $filename" if ! -e $filename; 12037e6b05aSAndre Fischer 12137e6b05aSAndre Fischer my $current_selector_value = 1; 12237e6b05aSAndre Fischer my @URLHeads = (); 12337e6b05aSAndre Fischer my @download_requests = (); 12437e6b05aSAndre Fischer 12537e6b05aSAndre Fischer open my $in, $filename; 12637e6b05aSAndre Fischer while (my $line = <$in>) 12737e6b05aSAndre Fischer { 12837e6b05aSAndre Fischer # Remove leading and trailing space and comments 12937e6b05aSAndre Fischer $line =~ s/^\s+//; 13037e6b05aSAndre Fischer $line =~ s/\s+$//; 13137e6b05aSAndre Fischer $line =~ s/\s*#.*$//; 13237e6b05aSAndre Fischer 13337e6b05aSAndre Fischer # Ignore empty lines. 13437e6b05aSAndre Fischer next if $line eq ""; 13537e6b05aSAndre Fischer 13637e6b05aSAndre Fischer # An "if" statement starts a new block. 13737e6b05aSAndre Fischer if ($line =~ /^\s*if\s*\(\s*(.*?)\s*\)\s*$/) 13837e6b05aSAndre Fischer { 13937e6b05aSAndre Fischer ProcessLastBlock(); 1406a22bca6SAndre Fischer 14137e6b05aSAndre Fischer $LocalEnvironment = { 'selector' => $1 }; 14237e6b05aSAndre Fischer } 14337e6b05aSAndre Fischer 14437e6b05aSAndre Fischer # Lines of the form name = value define a local variable. 14537e6b05aSAndre Fischer elsif ($line =~ /^\s*(\S+)\s*=\s*(.*?)\s*$/) 14637e6b05aSAndre Fischer { 14737e6b05aSAndre Fischer if (defined $LocalEnvironment) 14837e6b05aSAndre Fischer { 14937e6b05aSAndre Fischer $LocalEnvironment->{$1} = $2; 15037e6b05aSAndre Fischer } 15137e6b05aSAndre Fischer else 15237e6b05aSAndre Fischer { 15337e6b05aSAndre Fischer $GlobalEnvironment->{$1} = $2; 15437e6b05aSAndre Fischer } 15537e6b05aSAndre Fischer } 15637e6b05aSAndre Fischer else 15737e6b05aSAndre Fischer { 15837e6b05aSAndre Fischer die "can not parse line $line\n"; 15937e6b05aSAndre Fischer } 16037e6b05aSAndre Fischer } 16137e6b05aSAndre Fischer 16237e6b05aSAndre Fischer ProcessLastBlock(); 1636a22bca6SAndre Fischer 16437e6b05aSAndre Fischer Download(\@download_requests, \@URLHeads); 16537e6b05aSAndre Fischer} 16637e6b05aSAndre Fischer 16737e6b05aSAndre Fischer 16837e6b05aSAndre Fischer 16937e6b05aSAndre Fischer 17037e6b05aSAndre Fischer=head3 ProcessLastBlock 17137e6b05aSAndre Fischer 17237e6b05aSAndre Fischer Process the last definition of an external library. 17337e6b05aSAndre Fischer If there is not last block, true for the first "if" statement, then the call is ignored. 17437e6b05aSAndre Fischer 17537e6b05aSAndre Fischer=cut 17637e6b05aSAndre Fischersub ProcessLastBlock () 17737e6b05aSAndre Fischer{ 17837e6b05aSAndre Fischer # Return if no block is defined. 17937e6b05aSAndre Fischer return if ! defined $LocalEnvironment; 1806a22bca6SAndre Fischer 18137e6b05aSAndre Fischer # Ignore the block if the selector does not match. 18237e6b05aSAndre Fischer if ( ! EvaluateExpression(SubstituteVariables($LocalEnvironment->{'selector'}))) 18337e6b05aSAndre Fischer { 1846a22bca6SAndre Fischer printf("ignoring %s because its prerequisites are not fulfilled\n", GetValue('name')); 18537e6b05aSAndre Fischer } 18637e6b05aSAndre Fischer else 18737e6b05aSAndre Fischer { 18837e6b05aSAndre Fischer my $name = GetValue('name'); 18910e20387SAndre Fischer my $checksum = GetChecksum(); 19037e6b05aSAndre Fischer 1910aabba3aSAndre Fischer if ( ! IsPresent($name, $checksum)) 19210e20387SAndre Fischer { 19310e20387SAndre Fischer AddDownloadRequest($name, $checksum); 19437e6b05aSAndre Fischer } 19537e6b05aSAndre Fischer } 19637e6b05aSAndre Fischer} 19737e6b05aSAndre Fischer 19837e6b05aSAndre Fischer 19937e6b05aSAndre Fischer 20037e6b05aSAndre Fischer 20110e20387SAndre Fischer=head3 AddDownloadRequest($name, $checksum) 20237e6b05aSAndre Fischer 20337e6b05aSAndre Fischer Add a request for downloading the library $name to @Missing. 20437e6b05aSAndre Fischer Collect all available URL[1-9] variables as source URLs. 2056a22bca6SAndre Fischer 20637e6b05aSAndre Fischer=cut 20710e20387SAndre Fischersub AddDownloadRequest ($$) 20837e6b05aSAndre Fischer{ 20910e20387SAndre Fischer my ($name, $checksum) = @_; 21037e6b05aSAndre Fischer 21137e6b05aSAndre Fischer print "adding download request for $name\n"; 21237e6b05aSAndre Fischer 21337e6b05aSAndre Fischer my $urls = []; 21437e6b05aSAndre Fischer my $url = GetValue('URL'); 21537e6b05aSAndre Fischer push @$urls, SubstituteVariables($url) if (defined $url); 21637e6b05aSAndre Fischer for (my $i=1; $i<10; ++$i) 21737e6b05aSAndre Fischer { 21837e6b05aSAndre Fischer $url = GetValue('URL'.$i); 21937e6b05aSAndre Fischer next if ! defined $url; 22037e6b05aSAndre Fischer push @$urls, SubstituteVariables($url); 22137e6b05aSAndre Fischer } 22237e6b05aSAndre Fischer 22310e20387SAndre Fischer push @Missing, [$name, $checksum, $urls]; 22410e20387SAndre Fischer} 22510e20387SAndre Fischer 22610e20387SAndre Fischer 22710e20387SAndre Fischer 22810e20387SAndre Fischer 22910e20387SAndre Fischer=head3 GetChecksum() 23010e20387SAndre Fischer 23110e20387SAndre Fischer When either MD5 or SHA1 are variables in the current scope then return 23210e20387SAndre Fischer a reference to a hash with two entries: 23310e20387SAndre Fischer 'type' is either 'MD5' or 'SHA1', the type or algorithm of the checksum, 23410e20387SAndre Fischer 'value' is the actual checksum 23510e20387SAndre Fischer Otherwise undef is returned. 23610e20387SAndre Fischer 23710e20387SAndre Fischer=cut 23810e20387SAndre Fischersub GetChecksum() 23910e20387SAndre Fischer{ 24010e20387SAndre Fischer my $checksum = GetValue("MD5"); 24110e20387SAndre Fischer if (defined $checksum && $checksum ne "") 24210e20387SAndre Fischer { 24310e20387SAndre Fischer return { 'type' => 'MD5', 'value' => $checksum }; 24410e20387SAndre Fischer } 24510e20387SAndre Fischer elsif (defined ($checksum=GetValue("SHA1")) && $checksum ne "") 24610e20387SAndre Fischer { 24710e20387SAndre Fischer return { 'type' => 'SHA1', 'value' => $checksum }; 24810e20387SAndre Fischer } 24910e20387SAndre Fischer else 25010e20387SAndre Fischer { 25110e20387SAndre Fischer return undef; 25210e20387SAndre Fischer } 25337e6b05aSAndre Fischer} 25437e6b05aSAndre Fischer 25537e6b05aSAndre Fischer 25637e6b05aSAndre Fischer 25737e6b05aSAndre Fischer 25837e6b05aSAndre Fischer=head3 GetValue($variable_name) 25937e6b05aSAndre Fischer 26037e6b05aSAndre Fischer Return the value of the variable with name $variable_name from the local 26137e6b05aSAndre Fischer environment or, if not defined there, the global environment. 26237e6b05aSAndre Fischer 26337e6b05aSAndre Fischer=cut 26437e6b05aSAndre Fischersub GetValue ($) 26537e6b05aSAndre Fischer{ 26637e6b05aSAndre Fischer my $variable_name = shift; 26737e6b05aSAndre Fischer 26837e6b05aSAndre Fischer my $candidate = $LocalEnvironment->{$variable_name}; 26937e6b05aSAndre Fischer return $candidate if defined $candidate; 27037e6b05aSAndre Fischer 27137e6b05aSAndre Fischer return $GlobalEnvironment->{$variable_name}; 27237e6b05aSAndre Fischer} 27337e6b05aSAndre Fischer 27437e6b05aSAndre Fischer 27537e6b05aSAndre Fischer 27637e6b05aSAndre Fischer=head3 SubstituteVariables($text) 27737e6b05aSAndre Fischer 27837e6b05aSAndre Fischer Replace all references to variables in $text with the respective variable values. 27937e6b05aSAndre Fischer This is done repeatedly until no variable reference remains. 2806a22bca6SAndre Fischer 28137e6b05aSAndre Fischer=cut 28237e6b05aSAndre Fischersub SubstituteVariables ($) 28337e6b05aSAndre Fischer{ 28437e6b05aSAndre Fischer my $text = shift; 28537e6b05aSAndre Fischer 28637e6b05aSAndre Fischer my $infinite_recursion_guard = 100; 28737e6b05aSAndre Fischer while ($text =~ /^(.*?)\$\(([^)]+)\)(.*)$/) 28837e6b05aSAndre Fischer { 28937e6b05aSAndre Fischer my ($head,$name,$tail) = ($1,$2,$3); 29037e6b05aSAndre Fischer my $value = GetValue($name); 2910aabba3aSAndre Fischer die "can not evaluate variable $name" if ! defined $value; 29237e6b05aSAndre Fischer $text = $head.$value.$tail; 29337e6b05aSAndre Fischer 29437e6b05aSAndre Fischer die "(probably) detected an infinite recursion in variable definitions" if --$infinite_recursion_guard<=0; 29537e6b05aSAndre Fischer } 29637e6b05aSAndre Fischer 29737e6b05aSAndre Fischer return $text; 29837e6b05aSAndre Fischer} 29937e6b05aSAndre Fischer 30037e6b05aSAndre Fischer 30137e6b05aSAndre Fischer 30237e6b05aSAndre Fischer 30337e6b05aSAndre Fischer=head3 EvaluateExpression($expression) 30437e6b05aSAndre Fischer 30537e6b05aSAndre Fischer Evaluate the $expression of an "if" statement to either 0 or 1. It can 30637e6b05aSAndre Fischer be a single term (see EvaluateTerm for a description), or several terms 3076a22bca6SAndre Fischer separated by either all ||s or &&s. A term can also be an expression 3086a22bca6SAndre Fischer enclosed in parantheses. 3096a22bca6SAndre Fischer 31037e6b05aSAndre Fischer=cut 31137e6b05aSAndre Fischersub EvaluateExpression ($) 31237e6b05aSAndre Fischer{ 31337e6b05aSAndre Fischer my $expression = shift; 31437e6b05aSAndre Fischer 3156a22bca6SAndre Fischer # Evaluate sub expressions enclosed in parantheses. 3166a22bca6SAndre Fischer while ($expression =~ /^(.*)\(([^\(\)]+)\)(.*)$/) 3176a22bca6SAndre Fischer { 3186a22bca6SAndre Fischer $expression = $1 . (EvaluateExpression($2) ? " true " : " false ") . $3; 3196a22bca6SAndre Fischer } 3206a22bca6SAndre Fischer 32137e6b05aSAndre Fischer if ($expression =~ /&&/ && $expression =~ /\|\|/) 32237e6b05aSAndre Fischer { 3236a22bca6SAndre Fischer die "expression can contain either && or || but not both at the same time"; 32437e6b05aSAndre Fischer } 32537e6b05aSAndre Fischer elsif ($expression =~ /&&/) 32637e6b05aSAndre Fischer { 32737e6b05aSAndre Fischer foreach my $term (split (/\s*&&\s*/,$expression)) 32837e6b05aSAndre Fischer { 32937e6b05aSAndre Fischer return 0 if ! EvaluateTerm($term); 33037e6b05aSAndre Fischer } 33137e6b05aSAndre Fischer return 1; 33237e6b05aSAndre Fischer } 33337e6b05aSAndre Fischer elsif ($expression =~ /\|\|/) 33437e6b05aSAndre Fischer { 33537e6b05aSAndre Fischer foreach my $term (split (/\s*\|\|\s*/,$expression)) 33637e6b05aSAndre Fischer { 33737e6b05aSAndre Fischer return 1 if EvaluateTerm($term); 33837e6b05aSAndre Fischer } 33937e6b05aSAndre Fischer return 0; 34037e6b05aSAndre Fischer } 34137e6b05aSAndre Fischer else 34237e6b05aSAndre Fischer { 34337e6b05aSAndre Fischer return EvaluateTerm($expression); 34437e6b05aSAndre Fischer } 34537e6b05aSAndre Fischer} 34637e6b05aSAndre Fischer 34737e6b05aSAndre Fischer 34837e6b05aSAndre Fischer 34937e6b05aSAndre Fischer 35037e6b05aSAndre Fischer=head3 EvaluateTerm($term) 35137e6b05aSAndre Fischer 35237e6b05aSAndre Fischer Evaluate the $term to either 0 or 1. 35337e6b05aSAndre Fischer A term is either the literal "true", which evaluates to 1, or an expression 35437e6b05aSAndre Fischer of the form NAME=VALUE or NAME!=VALUE. NAME is the name of an environment 35537e6b05aSAndre Fischer variable and VALUE any string. VALUE may be empty. 3566a22bca6SAndre Fischer 35737e6b05aSAndre Fischer=cut 35837e6b05aSAndre Fischersub EvaluateTerm ($) 35937e6b05aSAndre Fischer{ 36037e6b05aSAndre Fischer my $term = shift; 36137e6b05aSAndre Fischer 3626a22bca6SAndre Fischer if ($term =~ /^\s*([a-zA-Z_0-9]+)\s*(==|!=)\s*(.*)\s*$/) 36337e6b05aSAndre Fischer { 36437e6b05aSAndre Fischer my ($variable_name, $operator, $given_value) = ($1,$2,$3); 36537e6b05aSAndre Fischer my $variable_value = $ENV{$variable_name}; 3666a22bca6SAndre Fischer $variable_value = "" if ! defined $variable_value; 36737e6b05aSAndre Fischer 3686a22bca6SAndre Fischer if ($operator eq "==") 36937e6b05aSAndre Fischer { 37037e6b05aSAndre Fischer return $variable_value eq $given_value; 37137e6b05aSAndre Fischer } 37237e6b05aSAndre Fischer elsif ($operator eq "!=") 37337e6b05aSAndre Fischer { 37437e6b05aSAndre Fischer return $variable_value ne $given_value; 37537e6b05aSAndre Fischer } 37637e6b05aSAndre Fischer else 37737e6b05aSAndre Fischer { 37837e6b05aSAndre Fischer die "unknown operator in term $term"; 37937e6b05aSAndre Fischer } 38037e6b05aSAndre Fischer } 38137e6b05aSAndre Fischer elsif ($term =~ /^\s*true\s*$/i) 38237e6b05aSAndre Fischer { 38337e6b05aSAndre Fischer return 1; 38437e6b05aSAndre Fischer } 3856a22bca6SAndre Fischer elsif ($term =~ /^\s*false\s*$/i) 3866a22bca6SAndre Fischer { 3876a22bca6SAndre Fischer return 0; 3886a22bca6SAndre Fischer } 38937e6b05aSAndre Fischer else 39037e6b05aSAndre Fischer { 39137e6b05aSAndre Fischer die "term $term is not of the form <environment-variable> (=|==) <value>"; 39237e6b05aSAndre Fischer } 39337e6b05aSAndre Fischer} 39437e6b05aSAndre Fischer 39537e6b05aSAndre Fischer 39637e6b05aSAndre Fischer 39737e6b05aSAndre Fischer 39810e20387SAndre Fischer=head IsPresent($name, $given_checksum) 39937e6b05aSAndre Fischer 40037e6b05aSAndre Fischer Check if an external library tar ball with the basename $name already 40137e6b05aSAndre Fischer exists in the target directory TARFILE_LOCATION. The basename is 40210e20387SAndre Fischer prefixed with the MD5 or SHA1 checksum. 40310e20387SAndre Fischer If the file exists then its checksum is compared to the given one. 4046a22bca6SAndre Fischer 40537e6b05aSAndre Fischer=cut 40637e6b05aSAndre Fischersub IsPresent ($$) 40737e6b05aSAndre Fischer{ 40810e20387SAndre Fischer my ($name, $given_checksum) = @_; 4096a22bca6SAndre Fischer 41010e20387SAndre Fischer my $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'}, $given_checksum->{'value'}."-".$name); 41110e20387SAndre Fischer return 0 unless -f $filename; 41237e6b05aSAndre Fischer 41310e20387SAndre Fischer # File exists. Check if its checksum is correct. 41410e20387SAndre Fischer my $checksum; 4150aabba3aSAndre Fischer if ( ! defined $given_checksum) 4160aabba3aSAndre Fischer { 4170aabba3aSAndre Fischer print "no checksum given, can not verify\n"; 4180aabba3aSAndre Fischer return 1; 4190aabba3aSAndre Fischer } 4200aabba3aSAndre Fischer elsif ($given_checksum->{'type'} eq "MD5") 42110e20387SAndre Fischer { 42210e20387SAndre Fischer my $md5 = Digest::MD5->new(); 42310e20387SAndre Fischer open my $in, $filename; 42410e20387SAndre Fischer $md5->addfile($in); 42510e20387SAndre Fischer $checksum = $md5->hexdigest(); 42610e20387SAndre Fischer } 42710e20387SAndre Fischer elsif ($given_checksum->{'type'} eq "SHA1") 42810e20387SAndre Fischer { 42910e20387SAndre Fischer my $sha1 = Digest::SHA->new("1"); 43010e20387SAndre Fischer open my $in, $filename; 43110e20387SAndre Fischer $sha1->addfile($in); 43210e20387SAndre Fischer $checksum = $sha1->hexdigest(); 43310e20387SAndre Fischer } 43410e20387SAndre Fischer else 43510e20387SAndre Fischer { 43610e20387SAndre Fischer die "unsupported checksum type (not MD5 or SHA1)"; 43710e20387SAndre Fischer } 43837e6b05aSAndre Fischer 43910e20387SAndre Fischer if ($given_checksum->{'value'} ne $checksum) 44037e6b05aSAndre Fischer { 44110e20387SAndre Fischer # Checksum does not match. Delete the file. 44210e20387SAndre Fischer print "$name exists, but checksum does not match => deleting\n"; 4430aabba3aSAndre Fischer unlink($filename); 44437e6b05aSAndre Fischer return 0; 44537e6b05aSAndre Fischer } 44637e6b05aSAndre Fischer else 44737e6b05aSAndre Fischer { 44810e20387SAndre Fischer printf("%s exists, %s checksum is OK\n", $name, $given_checksum->{'type'}); 44937e6b05aSAndre Fischer return 1; 45037e6b05aSAndre Fischer } 45137e6b05aSAndre Fischer} 45237e6b05aSAndre Fischer 45337e6b05aSAndre Fischer 45437e6b05aSAndre Fischer 45537e6b05aSAndre Fischer 45637e6b05aSAndre Fischer=head3 Download 45737e6b05aSAndre Fischer 45837e6b05aSAndre Fischer Download a set of files specified by @Missing. 45937e6b05aSAndre Fischer 46010e20387SAndre Fischer For http URLs there may be an optional checksum. If it is present then downloaded 46110e20387SAndre Fischer files that do not match that checksum lead to abortion of the current process. 46237e6b05aSAndre Fischer Files that have already been downloaded are not downloaded again. 4636a22bca6SAndre Fischer 46437e6b05aSAndre Fischer=cut 46537e6b05aSAndre Fischersub Download () 46637e6b05aSAndre Fischer{ 46737e6b05aSAndre Fischer my $download_path = $ENV{'TARFILE_LOCATION'}; 4686a22bca6SAndre Fischer 46937e6b05aSAndre Fischer if (scalar @Missing > 0) 47037e6b05aSAndre Fischer { 47137e6b05aSAndre Fischer printf("downloading %d missing tar ball%s to %s\n", 47237e6b05aSAndre Fischer scalar @Missing, scalar @Missing>0 ? "s" : "", 47337e6b05aSAndre Fischer $download_path); 47437e6b05aSAndre Fischer } 47537e6b05aSAndre Fischer else 47637e6b05aSAndre Fischer { 47737e6b05aSAndre Fischer print "all external libraries present\n"; 47837e6b05aSAndre Fischer return; 47937e6b05aSAndre Fischer } 4806a22bca6SAndre Fischer 48137e6b05aSAndre Fischer # Download the missing files. 48237e6b05aSAndre Fischer for my $item (@Missing) 48337e6b05aSAndre Fischer { 48410e20387SAndre Fischer my ($name, $checksum, $urls) = @$item; 4856a22bca6SAndre Fischer 48637e6b05aSAndre Fischer foreach my $url (@$urls) 48737e6b05aSAndre Fischer { 4880aabba3aSAndre Fischer last if DownloadFile( 4890aabba3aSAndre Fischer defined $checksum 4900aabba3aSAndre Fischer ? $checksum->{'value'}."-".$name 4910aabba3aSAndre Fischer : $name, 4920aabba3aSAndre Fischer $url, 4930aabba3aSAndre Fischer $checksum); 49437e6b05aSAndre Fischer } 49537e6b05aSAndre Fischer } 49637e6b05aSAndre Fischer} 49737e6b05aSAndre Fischer 49837e6b05aSAndre Fischer 49937e6b05aSAndre Fischer 50037e6b05aSAndre Fischer 50110e20387SAndre Fischer=head3 DownloadFile($name,$URL,$checksum) 50237e6b05aSAndre Fischer 50337e6b05aSAndre Fischer Download a single external library tarball. It origin is given by $URL. 50410e20387SAndre Fischer Its destination is $(TARFILE_LOCATION)/$checksum-$name. 5056a22bca6SAndre Fischer 50637e6b05aSAndre Fischer=cut 50737e6b05aSAndre Fischersub DownloadFile ($$$) 50837e6b05aSAndre Fischer{ 50937e6b05aSAndre Fischer my $name = shift; 51037e6b05aSAndre Fischer my $URL = shift; 51110e20387SAndre Fischer my $checksum = shift; 51237e6b05aSAndre Fischer 51337e6b05aSAndre Fischer my $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'}, $name); 51437e6b05aSAndre Fischer 51537e6b05aSAndre Fischer my $temporary_filename = $filename . ".part"; 51637e6b05aSAndre Fischer 51737e6b05aSAndre Fischer print "downloading to $temporary_filename\n"; 5180aabba3aSAndre Fischer my $out; 5190aabba3aSAndre Fischer open $out, ">$temporary_filename"; 52037e6b05aSAndre Fischer binmode($out); 52137e6b05aSAndre Fischer 52210e20387SAndre Fischer # Prepare checksum 52310e20387SAndre Fischer my $digest; 52410e20387SAndre Fischer if (defined $checksum && $checksum->{'type'} eq "SHA1") 52510e20387SAndre Fischer { 52610e20387SAndre Fischer # Use SHA1 only when explicitly requested (by the presence of a "SHA1=..." line.) 52710e20387SAndre Fischer $digest = Digest::SHA->new("1"); 52810e20387SAndre Fischer } 52910e20387SAndre Fischer elsif ( ! defined $checksum || $checksum->{'type'} eq "MD5") 53010e20387SAndre Fischer { 53110e20387SAndre Fischer # Use MD5 when explicitly requested or when no checksum type is given. 53210e20387SAndre Fischer $digest = Digest::MD5->new(); 53310e20387SAndre Fischer } 53410e20387SAndre Fischer else 53510e20387SAndre Fischer { 53610e20387SAndre Fischer die "checksum type ".$checksum->{'type'}." is not supported"; 53710e20387SAndre Fischer } 5386a22bca6SAndre Fischer 53937e6b05aSAndre Fischer # Download the extension. 540f253223cSAndre Fischer my $success = 0; 541f253223cSAndre Fischer if ($simple) 542f253223cSAndre Fischer { 543f253223cSAndre Fischer my $content = LWP::Simple::get($URL); 544f253223cSAndre Fischer $success = defined $content; 545f253223cSAndre Fischer if ($success) 546f253223cSAndre Fischer { 547f253223cSAndre Fischer open $out, ">$temporary_filename"; 548f253223cSAndre Fischer binmode($out); 549f253223cSAndre Fischer print $out $content; 550f253223cSAndre Fischer close($out); 551f253223cSAndre Fischer $digest->add($content); 552f253223cSAndre Fischer } 553f253223cSAndre Fischer else 554f253223cSAndre Fischer { 555f253223cSAndre Fischer print "download from $URL failed\n"; 556f253223cSAndre Fischer } 557f253223cSAndre Fischer } 558f253223cSAndre Fischer else 559f253223cSAndre Fischer { 560f253223cSAndre Fischer my $agent = LWP::UserAgent->new(); 561f253223cSAndre Fischer $agent->timeout(120); 562f253223cSAndre Fischer $agent->env_proxy; 563f253223cSAndre Fischer $agent->show_progress(1); 564f253223cSAndre Fischer my $last_was_redirect = 0; 565f253223cSAndre Fischer $agent->add_handler('response_redirect' 566f253223cSAndre Fischer => sub{ 567f253223cSAndre Fischer $last_was_redirect = 1; 568f253223cSAndre Fischer return; 569f253223cSAndre Fischer }); 570f253223cSAndre Fischer $agent->add_handler('response_data' 571f253223cSAndre Fischer => sub{ 572f253223cSAndre Fischer if ($last_was_redirect) 573f253223cSAndre Fischer { 574f253223cSAndre Fischer $last_was_redirect = 0; 575f253223cSAndre Fischer # Throw away the data we got so far. 576f253223cSAndre Fischer $digest->reset(); 577f253223cSAndre Fischer close $out; 578f253223cSAndre Fischer open $out, ">$temporary_filename"; 579f253223cSAndre Fischer binmode($out); 580f253223cSAndre Fischer } 581f253223cSAndre Fischer my($response,$agent,$h,$data)=@_; 582f253223cSAndre Fischer print $out $data; 583f253223cSAndre Fischer $digest->add($data); 584f253223cSAndre Fischer }); 585f253223cSAndre Fischer 586f253223cSAndre Fischer $success = $agent->get($URL)->is_success(); 587f253223cSAndre Fischer close $out; 588f253223cSAndre Fischer } 589f253223cSAndre Fischer 59086e1cf34SPedro Giffuni # When download was successful then check the checksum and rename the .part file 59137e6b05aSAndre Fischer # into the actual extension name. 592f253223cSAndre Fischer if ($success) 59337e6b05aSAndre Fischer { 59410e20387SAndre Fischer my $file_checksum = $digest->hexdigest(); 59510e20387SAndre Fischer if (defined $checksum) 59637e6b05aSAndre Fischer { 59710e20387SAndre Fischer if ($checksum->{'value'} eq $file_checksum) 59837e6b05aSAndre Fischer { 59910e20387SAndre Fischer printf("%s checksum is OK\n", $checksum->{'type'}); 60037e6b05aSAndre Fischer } 60137e6b05aSAndre Fischer else 60237e6b05aSAndre Fischer { 60337e6b05aSAndre Fischer unlink($temporary_filename); 60410e20387SAndre Fischer printf(" %s checksum does not match (%s instead of %s)\n", 605a90d3987SAndrea Pescetti $checksum->{'type'}, 60610e20387SAndre Fischer $file_checksum, 607a90d3987SAndrea Pescetti $checksum->{'value'}); 60837e6b05aSAndre Fischer return 0; 60937e6b05aSAndre Fischer } 61037e6b05aSAndre Fischer } 61137e6b05aSAndre Fischer else 61237e6b05aSAndre Fischer { 61310e20387SAndre Fischer # The datafile does not contain a checksum to match against. 61410e20387SAndre Fischer # Display the one that was calculated for the downloaded file so that 61510e20387SAndre Fischer # it can be integrated manually into the data file. 61610e20387SAndre Fischer printf("checksum not given, md5 of file is %s\n", $file_checksum); 61710e20387SAndre Fischer $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'}, $file_checksum . "-" . $name); 61837e6b05aSAndre Fischer } 6196a22bca6SAndre Fischer 62037e6b05aSAndre Fischer rename($temporary_filename, $filename) || die "can not rename $temporary_filename to $filename"; 62137e6b05aSAndre Fischer return 1; 62237e6b05aSAndre Fischer } 62337e6b05aSAndre Fischer else 62437e6b05aSAndre Fischer { 62537e6b05aSAndre Fischer unlink($temporary_filename); 62637e6b05aSAndre Fischer print " download failed\n"; 62737e6b05aSAndre Fischer return 0; 62837e6b05aSAndre Fischer } 62937e6b05aSAndre Fischer} 63037e6b05aSAndre Fischer 63137e6b05aSAndre Fischer 63237e6b05aSAndre Fischer 63337e6b05aSAndre Fischer 63437e6b05aSAndre Fischer=head3 CheckDownloadDestination () 63537e6b05aSAndre Fischer 63637e6b05aSAndre Fischer Make sure that the download destination $TARFILE_LOCATION does exist. If 63737e6b05aSAndre Fischer not, then the directory is created. 6386a22bca6SAndre Fischer 63937e6b05aSAndre Fischer=cut 64037e6b05aSAndre Fischersub CheckDownloadDestination () 64137e6b05aSAndre Fischer{ 64237e6b05aSAndre Fischer my $destination = $ENV{'TARFILE_LOCATION'}; 64337e6b05aSAndre Fischer die "ERROR: no destination defined! please set TARFILE_LOCATION!" if ($destination eq ""); 64437e6b05aSAndre Fischer 64537e6b05aSAndre Fischer if ( ! -d $destination) 64637e6b05aSAndre Fischer { 64737e6b05aSAndre Fischer File::Path::make_path($destination); 64837e6b05aSAndre Fischer die "ERROR: can't create \$TARFILE_LOCATION" if ! -d $destination; 64937e6b05aSAndre Fischer } 65037e6b05aSAndre Fischer} 65137e6b05aSAndre Fischer 65237e6b05aSAndre Fischer 65337e6b05aSAndre Fischer 65437e6b05aSAndre Fischer 65537e6b05aSAndre Fischer=head3 ProvideSpecialTarball ($url,$name,$name_converter) 65637e6b05aSAndre Fischer 65737e6b05aSAndre Fischer A few tarballs need special handling. That is done here. 6586a22bca6SAndre Fischer 65937e6b05aSAndre Fischer=cut 66037e6b05aSAndre Fischersub ProvideSpecialTarball ($$$) 66137e6b05aSAndre Fischer{ 66237e6b05aSAndre Fischer my $url = shift; 66337e6b05aSAndre Fischer my $name = shift; 66437e6b05aSAndre Fischer my $name_converter = shift; 66537e6b05aSAndre Fischer 66637e6b05aSAndre Fischer return unless defined $url && $url ne ""; 66737e6b05aSAndre Fischer 66837e6b05aSAndre Fischer # See if we can find the executable. 66937e6b05aSAndre Fischer my ($SOLARENV,$OUTPATH,$EXEEXT) = ($ENV{'SOLARENV'},$ENV{'OUTPATH'},$ENV{'EXEEXT'}); 67037e6b05aSAndre Fischer $SOLARENV = "" unless defined $SOLARENV; 67137e6b05aSAndre Fischer $OUTPATH = "" unless defined $OUTPATH; 67237e6b05aSAndre Fischer $EXEEXT = "" unless defined $EXEEXT; 67337e6b05aSAndre Fischer if (-x File::Spec->catfile($SOLARENV, $OUTPATH, "bin", $name.$EXEEXT)) 67437e6b05aSAndre Fischer { 67537e6b05aSAndre Fischer print "found $name executable\n"; 67637e6b05aSAndre Fischer return; 67737e6b05aSAndre Fischer } 67837e6b05aSAndre Fischer 67937e6b05aSAndre Fischer # Download the source from the URL. 68037e6b05aSAndre Fischer my $basename = basename(URI->new($url)->path()); 68137e6b05aSAndre Fischer die unless defined $basename; 68237e6b05aSAndre Fischer 68337e6b05aSAndre Fischer if (defined $name_converter) 68437e6b05aSAndre Fischer { 68537e6b05aSAndre Fischer $basename = &{$name_converter}($basename); 68637e6b05aSAndre Fischer } 6876a22bca6SAndre Fischer 68837e6b05aSAndre Fischer # Has the source tar ball already been downloaded? 68937e6b05aSAndre Fischer my @candidates = glob(File::Spec->catfile($ENV{'TARFILE_LOCATION'}, "*-" . $basename)); 69037e6b05aSAndre Fischer if (scalar @candidates > 0) 69137e6b05aSAndre Fischer { 69237e6b05aSAndre Fischer # Yes. 69337e6b05aSAndre Fischer print "$basename exists\n"; 69437e6b05aSAndre Fischer return; 69537e6b05aSAndre Fischer } 69637e6b05aSAndre Fischer else 69737e6b05aSAndre Fischer { 69837e6b05aSAndre Fischer # No, download it. 69937e6b05aSAndre Fischer print "downloading $basename\n"; 70037e6b05aSAndre Fischer DownloadFile($basename, $url, undef); 70137e6b05aSAndre Fischer } 70237e6b05aSAndre Fischer} 70337e6b05aSAndre Fischer 70437e6b05aSAndre Fischer 70537e6b05aSAndre Fischer 70637e6b05aSAndre Fischer 70737e6b05aSAndre Fischer 70837e6b05aSAndre Fischer# The main() functionality. 70937e6b05aSAndre Fischer 71037e6b05aSAndre Fischerdie "usage: $0 <data-file-name>" if scalar @ARGV != 1; 71137e6b05aSAndre Fischermy $data_file = $ARGV[0]; 71237e6b05aSAndre FischerCheckDownloadDestination(); 71337e6b05aSAndre FischerProcessDataFile($data_file); 71437e6b05aSAndre FischerProvideSpecialTarball($ENV{'DMAKE_URL'}, "dmake", undef); 71537e6b05aSAndre FischerProvideSpecialTarball( 71637e6b05aSAndre Fischer $ENV{'EPM_URL'}, 71737e6b05aSAndre Fischer "epm", 71837e6b05aSAndre Fischer sub{$_[0]=~s/-source//; return $_[0]}); 719