1c667dd47SPedro Giffuni#!/usr/bin/env perl
237e6b05aSAndre Fischer
396dbeebcSAndre Fischer#**************************************************************
4*e0f16157Smseidel#
596dbeebcSAndre Fischer#  Licensed to the Apache Software Foundation (ASF) under one
696dbeebcSAndre Fischer#  or more contributor license agreements.  See the NOTICE file
796dbeebcSAndre Fischer#  distributed with this work for additional information
896dbeebcSAndre Fischer#  regarding copyright ownership.  The ASF licenses this file
996dbeebcSAndre Fischer#  to you under the Apache License, Version 2.0 (the
1096dbeebcSAndre Fischer#  "License"); you may not use this file except in compliance
1196dbeebcSAndre Fischer#  with the License.  You may obtain a copy of the License at
12*e0f16157Smseidel#
1396dbeebcSAndre Fischer#    http://www.apache.org/licenses/LICENSE-2.0
14*e0f16157Smseidel#
1596dbeebcSAndre Fischer#  Unless required by applicable law or agreed to in writing,
1696dbeebcSAndre Fischer#  software distributed under the License is distributed on an
1796dbeebcSAndre Fischer#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
1896dbeebcSAndre Fischer#  KIND, either express or implied.  See the License for the
1996dbeebcSAndre Fischer#  specific language governing permissions and limitations
2096dbeebcSAndre Fischer#  under the License.
21*e0f16157Smseidel#
2296dbeebcSAndre Fischer#**************************************************************
2396dbeebcSAndre Fischer
2437e6b05aSAndre Fischer=head1 NAME
2537e6b05aSAndre Fischer
26*e0f16157Smseidel    download_external_libraries.pl - Load missing tarballs specified in main/external_deps.lst.
2737e6b05aSAndre Fischer
2837e6b05aSAndre Fischer=head1 SYNOPSIS
2937e6b05aSAndre Fischer
3037e6b05aSAndre Fischer    For downloading external libraries (typically from the main/bootstrap script):
316a22bca6SAndre Fischer
3237e6b05aSAndre Fischer    download_external_libraries(<data-file-name>);
3337e6b05aSAndre Fischer
3437e6b05aSAndre Fischer=head1 DESCRIPTION
3537e6b05aSAndre Fischer
36*e0f16157Smseidel    The contents of the main/external_deps.lst file are used to determine the
3737e6b05aSAndre Fischer    external library tarballs that are missing from ext_sources/.
3837e6b05aSAndre Fischer
3937e6b05aSAndre Fischer    Individual libraries can be ignored depending on the values of environment variables.
4037e6b05aSAndre Fischer
41*e0f16157Smseidel    Format of the main/external_deps.lst file:
4237e6b05aSAndre Fischer
4337e6b05aSAndre Fischer    The file is line based.
4437e6b05aSAndre Fischer    Comments start with a # and go to the end of the line and are ignored.
4537e6b05aSAndre Fischer    Lines that are empty or contain only spaces and/or comments are ignored.
4637e6b05aSAndre Fischer
4737e6b05aSAndre Fischer    All other lines can have one of two forms:
4837e6b05aSAndre Fischer    - A variable definition of the form <name>=<value>.
4937e6b05aSAndre Fischer    - A conditional block start in the form "if (<expression>)"
5037e6b05aSAndre Fischer
5137e6b05aSAndre Fischer    Variables defined in a conditional block are only visible in this block and
5237e6b05aSAndre Fischer    replace the definition of global variables and variables earlier in the same
5337e6b05aSAndre Fischer    block.
5437e6b05aSAndre Fischer    Some variables have special names:
5510e20387SAndre Fischer    - MD5 is the expected MD5 checksum of the library tarball.
5610e20387SAndre Fischer    - SHA1 is the expected SHA1 checksum of the library tarball.
57*e0f16157Smseidel    - URL1 to URL9 specify from where to download the tarball. The URLs are tried in order.
5810e20387SAndre Fischer      The first successful download (download completed and checksum match) stops the iteration.
5937e6b05aSAndre Fischer
6037e6b05aSAndre Fischer    Expressions are explained below in the comment of EvaluateExpression().
6137e6b05aSAndre Fischer
6237e6b05aSAndre Fischer    A library is only regarded if its conditional expression evaluates to 1.
6337e6b05aSAndre Fischer
6437e6b05aSAndre Fischer    Example:
6537e6b05aSAndre Fischer
6637e6b05aSAndre Fischer    DefaultSite=http://some-internet-site.org
6737e6b05aSAndre Fischer    if ( true )
6837e6b05aSAndre Fischer        MD5 = 0123456789abcdef0123456789abcdef
6937e6b05aSAndre Fischer        name = library-1.0.tar.gz
7037e6b05aSAndre Fischer        URL1 = http://some-other-internet-site.org/another-name.tgz
7137e6b05aSAndre Fischer        URL2 = $(DefaultSite)$(MD5)-$(name)
7237e6b05aSAndre Fischer
7337e6b05aSAndre Fischer    This tries to load a library first from some-other-internet-site.org and if
74*e0f16157Smseidel    that fails from some-internet-site.org. The library is stored as $(MD5)-$(name)
7537e6b05aSAndre Fischer    even when it is loaded as another-name.tgz.
7637e6b05aSAndre Fischer
7737e6b05aSAndre Fischer=cut
7837e6b05aSAndre Fischer
7937e6b05aSAndre Fischer
8037e6b05aSAndre Fischeruse strict;
8137e6b05aSAndre Fischer
8237e6b05aSAndre Fischeruse File::Spec;
8337e6b05aSAndre Fischeruse File::Path;
8437e6b05aSAndre Fischeruse File::Basename;
8537e6b05aSAndre Fischeruse Digest::MD5;
8610e20387SAndre Fischeruse Digest::SHA;
8737e6b05aSAndre Fischeruse URI;
8806aba1c9SAndrea Pescettiuse LWP::UserAgent;
8937e6b05aSAndre Fischer
9037e6b05aSAndre Fischermy $Debug = 1;
9137e6b05aSAndre Fischer
9237e6b05aSAndre Fischermy $LocalEnvironment = undef;
9337e6b05aSAndre Fischermy $GlobalEnvironment = {};
9437e6b05aSAndre Fischermy @Missing = ();
9537e6b05aSAndre Fischer
9637e6b05aSAndre Fischer
9737e6b05aSAndre Fischer
9837e6b05aSAndre Fischer
9937e6b05aSAndre Fischer=head3 ProcessDataFile
1006a22bca6SAndre Fischer
101*e0f16157Smseidel    Read the data file, typically named main/external_deps.lst, find the external
10237e6b05aSAndre Fischer    library tarballs that are not yet present in ext_sources/ and download them.
10337e6b05aSAndre Fischer
10437e6b05aSAndre Fischer=cut
10537e6b05aSAndre Fischersub ProcessDataFile ($)
10637e6b05aSAndre Fischer{
10737e6b05aSAndre Fischer    my $filename = shift;
10837e6b05aSAndre Fischer
10937e6b05aSAndre Fischer    my $destination = $ENV{'TARFILE_LOCATION'};
11037e6b05aSAndre Fischer
11137e6b05aSAndre Fischer    die "can not open data file $filename" if ! -e $filename;
11237e6b05aSAndre Fischer
11337e6b05aSAndre Fischer    my $current_selector_value = 1;
11437e6b05aSAndre Fischer    my @URLHeads = ();
11537e6b05aSAndre Fischer    my @download_requests = ();
11637e6b05aSAndre Fischer
11737e6b05aSAndre Fischer    open my $in, $filename;
11837e6b05aSAndre Fischer    while (my $line = <$in>)
11937e6b05aSAndre Fischer    {
12037e6b05aSAndre Fischer        # Remove leading and trailing space and comments
12137e6b05aSAndre Fischer        $line =~ s/^\s+//;
12237e6b05aSAndre Fischer        $line =~ s/\s+$//;
12337e6b05aSAndre Fischer        $line =~ s/\s*#.*$//;
12437e6b05aSAndre Fischer
12537e6b05aSAndre Fischer        # Ignore empty lines.
12637e6b05aSAndre Fischer        next if $line eq "";
12737e6b05aSAndre Fischer
12837e6b05aSAndre Fischer        # An "if" statement starts a new block.
12937e6b05aSAndre Fischer        if ($line =~ /^\s*if\s*\(\s*(.*?)\s*\)\s*$/)
13037e6b05aSAndre Fischer        {
13137e6b05aSAndre Fischer            ProcessLastBlock();
1326a22bca6SAndre Fischer
13337e6b05aSAndre Fischer            $LocalEnvironment = { 'selector' => $1 };
13437e6b05aSAndre Fischer        }
13537e6b05aSAndre Fischer
13637e6b05aSAndre Fischer        # Lines of the form name = value define a local variable.
13737e6b05aSAndre Fischer        elsif ($line =~ /^\s*(\S+)\s*=\s*(.*?)\s*$/)
13837e6b05aSAndre Fischer        {
13937e6b05aSAndre Fischer            if (defined $LocalEnvironment)
14037e6b05aSAndre Fischer            {
14137e6b05aSAndre Fischer                $LocalEnvironment->{$1} = $2;
14237e6b05aSAndre Fischer            }
14337e6b05aSAndre Fischer            else
14437e6b05aSAndre Fischer            {
14537e6b05aSAndre Fischer                $GlobalEnvironment->{$1} = $2;
14637e6b05aSAndre Fischer            }
14737e6b05aSAndre Fischer        }
14837e6b05aSAndre Fischer        else
14937e6b05aSAndre Fischer        {
15037e6b05aSAndre Fischer            die "can not parse line $line\n";
15137e6b05aSAndre Fischer        }
15237e6b05aSAndre Fischer    }
15337e6b05aSAndre Fischer
15437e6b05aSAndre Fischer    ProcessLastBlock();
1556a22bca6SAndre Fischer
15637e6b05aSAndre Fischer    Download(\@download_requests, \@URLHeads);
15737e6b05aSAndre Fischer}
15837e6b05aSAndre Fischer
15937e6b05aSAndre Fischer
16037e6b05aSAndre Fischer
16137e6b05aSAndre Fischer
16237e6b05aSAndre Fischer=head3 ProcessLastBlock
16337e6b05aSAndre Fischer
16437e6b05aSAndre Fischer    Process the last definition of an external library.
16537e6b05aSAndre Fischer    If there is not last block, true for the first "if" statement, then the call is ignored.
16637e6b05aSAndre Fischer
16737e6b05aSAndre Fischer=cut
16837e6b05aSAndre Fischersub ProcessLastBlock ()
16937e6b05aSAndre Fischer{
17037e6b05aSAndre Fischer    # Return if no block is defined.
17137e6b05aSAndre Fischer    return if ! defined $LocalEnvironment;
1726a22bca6SAndre Fischer
17337e6b05aSAndre Fischer    # Ignore the block if the selector does not match.
17437e6b05aSAndre Fischer    if ( ! EvaluateExpression(SubstituteVariables($LocalEnvironment->{'selector'})))
17537e6b05aSAndre Fischer    {
1766a22bca6SAndre Fischer        printf("ignoring %s because its prerequisites are not fulfilled\n", GetValue('name'));
17737e6b05aSAndre Fischer    }
17837e6b05aSAndre Fischer    else
17937e6b05aSAndre Fischer    {
18037e6b05aSAndre Fischer        my $name = GetValue('name');
18110e20387SAndre Fischer        my $checksum = GetChecksum();
18237e6b05aSAndre Fischer
1830aabba3aSAndre Fischer        if ( ! IsPresent($name, $checksum))
18410e20387SAndre Fischer        {
18510e20387SAndre Fischer            AddDownloadRequest($name, $checksum);
18637e6b05aSAndre Fischer        }
18737e6b05aSAndre Fischer    }
18837e6b05aSAndre Fischer}
18937e6b05aSAndre Fischer
19037e6b05aSAndre Fischer
19137e6b05aSAndre Fischer
19237e6b05aSAndre Fischer
19310e20387SAndre Fischer=head3 AddDownloadRequest($name, $checksum)
19437e6b05aSAndre Fischer
19537e6b05aSAndre Fischer    Add a request for downloading the library $name to @Missing.
19637e6b05aSAndre Fischer    Collect all available URL[1-9] variables as source URLs.
1976a22bca6SAndre Fischer
19837e6b05aSAndre Fischer=cut
19910e20387SAndre Fischersub AddDownloadRequest ($$)
20037e6b05aSAndre Fischer{
20110e20387SAndre Fischer    my ($name, $checksum) = @_;
20237e6b05aSAndre Fischer
20337e6b05aSAndre Fischer    print "adding download request for $name\n";
20437e6b05aSAndre Fischer
20537e6b05aSAndre Fischer    my $urls = [];
20637e6b05aSAndre Fischer    my $url = GetValue('URL');
20737e6b05aSAndre Fischer    push @$urls, SubstituteVariables($url) if (defined $url);
20837e6b05aSAndre Fischer    for (my $i=1; $i<10; ++$i)
20937e6b05aSAndre Fischer    {
21037e6b05aSAndre Fischer        $url = GetValue('URL'.$i);
21137e6b05aSAndre Fischer        next if ! defined $url;
21237e6b05aSAndre Fischer        push @$urls, SubstituteVariables($url);
21337e6b05aSAndre Fischer    }
21437e6b05aSAndre Fischer
21510e20387SAndre Fischer    push @Missing, [$name, $checksum, $urls];
21610e20387SAndre Fischer}
21710e20387SAndre Fischer
21810e20387SAndre Fischer
21910e20387SAndre Fischer
22010e20387SAndre Fischer
22110e20387SAndre Fischer=head3 GetChecksum()
22210e20387SAndre Fischer
22310e20387SAndre Fischer    When either MD5 or SHA1 are variables in the current scope then return
22410e20387SAndre Fischer    a reference to a hash with two entries:
22510e20387SAndre Fischer        'type' is either 'MD5' or 'SHA1', the type or algorithm of the checksum,
22610e20387SAndre Fischer        'value' is the actual checksum
22710e20387SAndre Fischer    Otherwise undef is returned.
228*e0f16157Smseidel
22910e20387SAndre Fischer=cut
23010e20387SAndre Fischersub GetChecksum()
23110e20387SAndre Fischer{
23210e20387SAndre Fischer    my $checksum = GetValue("MD5");
23310e20387SAndre Fischer    if (defined $checksum && $checksum ne "")
23410e20387SAndre Fischer    {
23510e20387SAndre Fischer        return { 'type' => 'MD5', 'value' => $checksum };
23610e20387SAndre Fischer    }
23710e20387SAndre Fischer    elsif (defined ($checksum=GetValue("SHA1")) && $checksum ne "")
23810e20387SAndre Fischer    {
23910e20387SAndre Fischer        return { 'type' => 'SHA1', 'value' => $checksum };
24010e20387SAndre Fischer    }
24110e20387SAndre Fischer    else
24210e20387SAndre Fischer    {
24310e20387SAndre Fischer        return undef;
24410e20387SAndre Fischer    }
24537e6b05aSAndre Fischer}
24637e6b05aSAndre Fischer
24737e6b05aSAndre Fischer
24837e6b05aSAndre Fischer
24937e6b05aSAndre Fischer
25037e6b05aSAndre Fischer=head3 GetValue($variable_name)
25137e6b05aSAndre Fischer
25237e6b05aSAndre Fischer    Return the value of the variable with name $variable_name from the local
25337e6b05aSAndre Fischer    environment or, if not defined there, the global environment.
25437e6b05aSAndre Fischer
25537e6b05aSAndre Fischer=cut
25637e6b05aSAndre Fischersub GetValue ($)
25737e6b05aSAndre Fischer{
25837e6b05aSAndre Fischer    my $variable_name = shift;
25937e6b05aSAndre Fischer
26037e6b05aSAndre Fischer    my $candidate = $LocalEnvironment->{$variable_name};
26137e6b05aSAndre Fischer    return $candidate if defined $candidate;
26237e6b05aSAndre Fischer
26337e6b05aSAndre Fischer    return $GlobalEnvironment->{$variable_name};
26437e6b05aSAndre Fischer}
26537e6b05aSAndre Fischer
26637e6b05aSAndre Fischer
26737e6b05aSAndre Fischer
26837e6b05aSAndre Fischer=head3 SubstituteVariables($text)
26937e6b05aSAndre Fischer
27037e6b05aSAndre Fischer    Replace all references to variables in $text with the respective variable values.
27137e6b05aSAndre Fischer    This is done repeatedly until no variable reference remains.
2726a22bca6SAndre Fischer
27337e6b05aSAndre Fischer=cut
27437e6b05aSAndre Fischersub SubstituteVariables ($)
27537e6b05aSAndre Fischer{
27637e6b05aSAndre Fischer    my $text = shift;
27737e6b05aSAndre Fischer
27837e6b05aSAndre Fischer    my $infinite_recursion_guard = 100;
27937e6b05aSAndre Fischer    while ($text =~ /^(.*?)\$\(([^)]+)\)(.*)$/)
28037e6b05aSAndre Fischer    {
28137e6b05aSAndre Fischer        my ($head,$name,$tail) = ($1,$2,$3);
28237e6b05aSAndre Fischer        my $value = GetValue($name);
2830aabba3aSAndre Fischer        die "can not evaluate variable $name" if ! defined $value;
28437e6b05aSAndre Fischer        $text = $head.$value.$tail;
28537e6b05aSAndre Fischer
28637e6b05aSAndre Fischer        die "(probably) detected an infinite recursion in variable definitions" if --$infinite_recursion_guard<=0;
28737e6b05aSAndre Fischer    }
28837e6b05aSAndre Fischer
28937e6b05aSAndre Fischer    return $text;
29037e6b05aSAndre Fischer}
29137e6b05aSAndre Fischer
29237e6b05aSAndre Fischer
29337e6b05aSAndre Fischer
29437e6b05aSAndre Fischer
29537e6b05aSAndre Fischer=head3 EvaluateExpression($expression)
29637e6b05aSAndre Fischer
297*e0f16157Smseidel    Evaluate the $expression of an "if" statement to either 0 or 1. It can
29837e6b05aSAndre Fischer    be a single term (see EvaluateTerm for a description), or several terms
299*e0f16157Smseidel    separated by either all ||s or &&s. A term can also be an expression
300*e0f16157Smseidel    enclosed in parentheses.
3016a22bca6SAndre Fischer
30237e6b05aSAndre Fischer=cut
30337e6b05aSAndre Fischersub EvaluateExpression ($)
30437e6b05aSAndre Fischer{
30537e6b05aSAndre Fischer    my $expression = shift;
30637e6b05aSAndre Fischer
307*e0f16157Smseidel    # Evaluate sub expressions enclosed in parentheses.
3086a22bca6SAndre Fischer    while ($expression =~ /^(.*)\(([^\(\)]+)\)(.*)$/)
3096a22bca6SAndre Fischer    {
3106a22bca6SAndre Fischer        $expression = $1 . (EvaluateExpression($2) ? " true " : " false ") . $3;
3116a22bca6SAndre Fischer    }
3126a22bca6SAndre Fischer
31337e6b05aSAndre Fischer    if ($expression =~ /&&/ && $expression =~ /\|\|/)
31437e6b05aSAndre Fischer    {
3156a22bca6SAndre Fischer        die "expression can contain either && or || but not both at the same time";
31637e6b05aSAndre Fischer    }
31737e6b05aSAndre Fischer    elsif ($expression =~ /&&/)
31837e6b05aSAndre Fischer    {
31937e6b05aSAndre Fischer        foreach my $term (split (/\s*&&\s*/,$expression))
32037e6b05aSAndre Fischer        {
32137e6b05aSAndre Fischer            return 0 if ! EvaluateTerm($term);
32237e6b05aSAndre Fischer        }
32337e6b05aSAndre Fischer        return 1;
32437e6b05aSAndre Fischer    }
32537e6b05aSAndre Fischer    elsif ($expression =~ /\|\|/)
32637e6b05aSAndre Fischer    {
32737e6b05aSAndre Fischer        foreach my $term (split (/\s*\|\|\s*/,$expression))
32837e6b05aSAndre Fischer        {
32937e6b05aSAndre Fischer            return 1 if EvaluateTerm($term);
33037e6b05aSAndre Fischer        }
33137e6b05aSAndre Fischer        return 0;
33237e6b05aSAndre Fischer    }
33337e6b05aSAndre Fischer    else
33437e6b05aSAndre Fischer    {
33537e6b05aSAndre Fischer        return EvaluateTerm($expression);
33637e6b05aSAndre Fischer    }
33737e6b05aSAndre Fischer}
33837e6b05aSAndre Fischer
33937e6b05aSAndre Fischer
34037e6b05aSAndre Fischer
34137e6b05aSAndre Fischer
34237e6b05aSAndre Fischer=head3 EvaluateTerm($term)
34337e6b05aSAndre Fischer
34437e6b05aSAndre Fischer    Evaluate the $term to either 0 or 1.
34537e6b05aSAndre Fischer    A term is either the literal "true", which evaluates to 1, or an expression
346*e0f16157Smseidel    of the form NAME=VALUE or NAME!=VALUE. NAME is the name of an environment
347*e0f16157Smseidel    variable and VALUE any string. VALUE may be empty.
3486a22bca6SAndre Fischer
34937e6b05aSAndre Fischer=cut
35037e6b05aSAndre Fischersub EvaluateTerm ($)
35137e6b05aSAndre Fischer{
35237e6b05aSAndre Fischer    my $term = shift;
35337e6b05aSAndre Fischer
3546a22bca6SAndre Fischer    if ($term =~ /^\s*([a-zA-Z_0-9]+)\s*(==|!=)\s*(.*)\s*$/)
35537e6b05aSAndre Fischer    {
35637e6b05aSAndre Fischer        my ($variable_name, $operator, $given_value) = ($1,$2,$3);
35737e6b05aSAndre Fischer        my $variable_value = $ENV{$variable_name};
3586a22bca6SAndre Fischer        $variable_value = "" if ! defined $variable_value;
35937e6b05aSAndre Fischer
3606a22bca6SAndre Fischer        if ($operator eq "==")
36137e6b05aSAndre Fischer        {
36237e6b05aSAndre Fischer            return $variable_value eq $given_value;
36337e6b05aSAndre Fischer        }
36437e6b05aSAndre Fischer        elsif ($operator eq "!=")
36537e6b05aSAndre Fischer        {
36637e6b05aSAndre Fischer            return $variable_value ne $given_value;
36737e6b05aSAndre Fischer        }
36837e6b05aSAndre Fischer        else
36937e6b05aSAndre Fischer        {
37037e6b05aSAndre Fischer            die "unknown operator in term $term";
37137e6b05aSAndre Fischer        }
37237e6b05aSAndre Fischer    }
37337e6b05aSAndre Fischer    elsif ($term =~ /^\s*true\s*$/i)
37437e6b05aSAndre Fischer    {
37537e6b05aSAndre Fischer        return 1;
37637e6b05aSAndre Fischer    }
3776a22bca6SAndre Fischer    elsif ($term =~ /^\s*false\s*$/i)
3786a22bca6SAndre Fischer    {
3796a22bca6SAndre Fischer        return 0;
3806a22bca6SAndre Fischer    }
38137e6b05aSAndre Fischer    else
38237e6b05aSAndre Fischer    {
38337e6b05aSAndre Fischer        die "term $term is not of the form <environment-variable> (=|==) <value>";
38437e6b05aSAndre Fischer    }
38537e6b05aSAndre Fischer}
38637e6b05aSAndre Fischer
38737e6b05aSAndre Fischer
38837e6b05aSAndre Fischer
38937e6b05aSAndre Fischer
39010e20387SAndre Fischer=head IsPresent($name, $given_checksum)
39137e6b05aSAndre Fischer
39237e6b05aSAndre Fischer    Check if an external library tar ball with the basename $name already
393*e0f16157Smseidel    exists in the target directory TARFILE_LOCATION. The basename is
39410e20387SAndre Fischer    prefixed with the MD5 or SHA1 checksum.
39510e20387SAndre Fischer    If the file exists then its checksum is compared to the given one.
3966a22bca6SAndre Fischer
39737e6b05aSAndre Fischer=cut
39837e6b05aSAndre Fischersub IsPresent ($$)
39937e6b05aSAndre Fischer{
40010e20387SAndre Fischer    my ($name, $given_checksum) = @_;
4016a22bca6SAndre Fischer
40210e20387SAndre Fischer    my $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'}, $given_checksum->{'value'}."-".$name);
40310e20387SAndre Fischer    return 0 unless -f $filename;
40437e6b05aSAndre Fischer
405*e0f16157Smseidel    # File exists. Check if its checksum is correct.
40610e20387SAndre Fischer    my $checksum;
4070aabba3aSAndre Fischer    if ( ! defined $given_checksum)
4080aabba3aSAndre Fischer    {
4090aabba3aSAndre Fischer        print "no checksum given, can not verify\n";
4100aabba3aSAndre Fischer        return 1;
4110aabba3aSAndre Fischer    }
4120aabba3aSAndre Fischer    elsif ($given_checksum->{'type'} eq "MD5")
41310e20387SAndre Fischer    {
41410e20387SAndre Fischer        my $md5 = Digest::MD5->new();
41510e20387SAndre Fischer        open my $in, $filename;
41610e20387SAndre Fischer        $md5->addfile($in);
41710e20387SAndre Fischer        $checksum = $md5->hexdigest();
41810e20387SAndre Fischer    }
41910e20387SAndre Fischer    elsif ($given_checksum->{'type'} eq "SHA1")
42010e20387SAndre Fischer    {
42110e20387SAndre Fischer        my $sha1 = Digest::SHA->new("1");
42210e20387SAndre Fischer        open my $in, $filename;
42310e20387SAndre Fischer        $sha1->addfile($in);
42410e20387SAndre Fischer        $checksum = $sha1->hexdigest();
42510e20387SAndre Fischer    }
42610e20387SAndre Fischer    else
42710e20387SAndre Fischer    {
42810e20387SAndre Fischer        die "unsupported checksum type (not MD5 or SHA1)";
42910e20387SAndre Fischer    }
43037e6b05aSAndre Fischer
43110e20387SAndre Fischer    if ($given_checksum->{'value'} ne $checksum)
43237e6b05aSAndre Fischer    {
433*e0f16157Smseidel        # Checksum does not match. Delete the file.
43410e20387SAndre Fischer        print "$name exists, but checksum does not match => deleting\n";
4350aabba3aSAndre Fischer        unlink($filename);
43637e6b05aSAndre Fischer        return 0;
43737e6b05aSAndre Fischer    }
43837e6b05aSAndre Fischer    else
43937e6b05aSAndre Fischer    {
44010e20387SAndre Fischer        printf("%s exists, %s checksum is OK\n", $name, $given_checksum->{'type'});
44137e6b05aSAndre Fischer        return 1;
44237e6b05aSAndre Fischer    }
44337e6b05aSAndre Fischer}
44437e6b05aSAndre Fischer
44537e6b05aSAndre Fischer
44637e6b05aSAndre Fischer
44737e6b05aSAndre Fischer
44837e6b05aSAndre Fischer=head3 Download
44937e6b05aSAndre Fischer
45037e6b05aSAndre Fischer    Download a set of files specified by @Missing.
45137e6b05aSAndre Fischer
452*e0f16157Smseidel    For http URLs there may be an optional checksum. If it is present then downloaded
45310e20387SAndre Fischer    files that do not match that checksum lead to abortion of the current process.
45437e6b05aSAndre Fischer    Files that have already been downloaded are not downloaded again.
4556a22bca6SAndre Fischer
45637e6b05aSAndre Fischer=cut
45737e6b05aSAndre Fischersub Download ()
45837e6b05aSAndre Fischer{
45937e6b05aSAndre Fischer    my $download_path = $ENV{'TARFILE_LOCATION'};
4606a22bca6SAndre Fischer
46137e6b05aSAndre Fischer    if (scalar @Missing > 0)
46237e6b05aSAndre Fischer    {
46337e6b05aSAndre Fischer        printf("downloading %d missing tar ball%s to %s\n",
46437e6b05aSAndre Fischer               scalar @Missing, scalar @Missing>0 ? "s" : "",
46537e6b05aSAndre Fischer               $download_path);
46637e6b05aSAndre Fischer    }
46737e6b05aSAndre Fischer    else
46837e6b05aSAndre Fischer    {
46937e6b05aSAndre Fischer        print "all external libraries present\n";
47037e6b05aSAndre Fischer        return;
47137e6b05aSAndre Fischer    }
4726a22bca6SAndre Fischer
47337e6b05aSAndre Fischer    # Download the missing files.
47483e0440aSDamjan Jovanovic    my $all_downloaded = 1;
47537e6b05aSAndre Fischer    for my $item (@Missing)
47637e6b05aSAndre Fischer    {
47710e20387SAndre Fischer        my ($name, $checksum, $urls) = @$item;
4786a22bca6SAndre Fischer
47983e0440aSDamjan Jovanovic        my $downloaded = 0;
48037e6b05aSAndre Fischer        foreach my $url (@$urls)
48137e6b05aSAndre Fischer        {
48283e0440aSDamjan Jovanovic            $downloaded = DownloadFile(
4830aabba3aSAndre Fischer                defined $checksum
4840aabba3aSAndre Fischer                    ? $checksum->{'value'}."-".$name
4850aabba3aSAndre Fischer                    : $name,
4860aabba3aSAndre Fischer                $url,
4870aabba3aSAndre Fischer                $checksum);
48883e0440aSDamjan Jovanovic            last if $downloaded
48937e6b05aSAndre Fischer        }
49083e0440aSDamjan Jovanovic        $all_downloaded &&= $downloaded;
49137e6b05aSAndre Fischer    }
49283e0440aSDamjan Jovanovic    die "some needed files could not be downloaded!" if !$all_downloaded;
49337e6b05aSAndre Fischer}
49437e6b05aSAndre Fischer
49537e6b05aSAndre Fischer
49637e6b05aSAndre Fischer
49737e6b05aSAndre Fischer
49810e20387SAndre Fischer=head3 DownloadFile($name,$URL,$checksum)
49937e6b05aSAndre Fischer
500*e0f16157Smseidel    Download a single external library tarball. Its origin is given by $URL.
50110e20387SAndre Fischer    Its destination is $(TARFILE_LOCATION)/$checksum-$name.
5026a22bca6SAndre Fischer
50337e6b05aSAndre Fischer=cut
50437e6b05aSAndre Fischersub DownloadFile ($$$)
50537e6b05aSAndre Fischer{
50637e6b05aSAndre Fischer    my $name = shift;
50737e6b05aSAndre Fischer    my $URL = shift;
50810e20387SAndre Fischer    my $checksum = shift;
50937e6b05aSAndre Fischer
510c0f6b924Sarielch    my $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'}, $name);
511c0f6b924Sarielch
512c0f6b924Sarielch    my $temporary_filename = $filename . ".part";
513c0f6b924Sarielch
514c0f6b924Sarielch    print "downloading to $temporary_filename\n";
515c0f6b924Sarielch    my $out;
516c0f6b924Sarielch    open $out, ">$temporary_filename";
517c0f6b924Sarielch    binmode($out);
518c0f6b924Sarielch
519c0f6b924Sarielch    # Prepare checksum
520c0f6b924Sarielch    my $digest;
521c0f6b924Sarielch    if (defined $checksum && $checksum->{'type'} eq "SHA1")
522c0f6b924Sarielch    {
523c0f6b924Sarielch        # Use SHA1 only when explicitly requested (by the presence of a "SHA1=..." line.)
524c0f6b924Sarielch        $digest = Digest::SHA->new("1");
525c0f6b924Sarielch    }
526c0f6b924Sarielch    elsif ( ! defined $checksum || $checksum->{'type'} eq "MD5")
527c0f6b924Sarielch    {
528c0f6b924Sarielch        # Use MD5 when explicitly requested or when no checksum type is given.
529c0f6b924Sarielch        $digest = Digest::MD5->new();
53010e20387SAndre Fischer    }
53110e20387SAndre Fischer    else
53210e20387SAndre Fischer    {
533c0f6b924Sarielch        die "checksum type ".$checksum->{'type'}." is not supported";
53410e20387SAndre Fischer    }
5356a22bca6SAndre Fischer
536c0f6b924Sarielch    # Download the extension.
537c0f6b924Sarielch    my $success = 0;
538c0f6b924Sarielch
539c0f6b924Sarielch    my $agent = LWP::UserAgent->new();
540c0f6b924Sarielch    $agent->env_proxy;
541c0f6b924Sarielch    my $response = $agent->get($URL);
542c0f6b924Sarielch
543c0f6b924Sarielch    $success = $response->is_success;
544c0f6b924Sarielch    if ($success)
545f253223cSAndre Fischer    {
546c0f6b924Sarielch        my $content = $response->content;
547c0f6b924Sarielch        open $out, ">$temporary_filename";
548c0f6b924Sarielch        binmode($out);
549c0f6b924Sarielch        print $out $content;
550c0f6b924Sarielch        $digest->add($content);
55137e6b05aSAndre Fischer    }
552c0f6b924Sarielch    else
55337e6b05aSAndre Fischer    {
554c0f6b924Sarielch        print "download from $URL failed (" . $response->status_line . ")\n";
555c0f6b924Sarielch    }
556c0f6b924Sarielch    close($out);
557c0f6b924Sarielch
558c0f6b924Sarielch    # When download was successful then check the checksum and rename the .part file
559c0f6b924Sarielch    # into the actual extension name.
560c0f6b924Sarielch    if ($success)
561c0f6b924Sarielch    {
562c0f6b924Sarielch        my $file_checksum = $digest->hexdigest();
563c0f6b924Sarielch        if (defined $checksum)
564c0f6b924Sarielch        {
565c0f6b924Sarielch            if ($checksum->{'value'} eq $file_checksum)
566c0f6b924Sarielch            {
567c0f6b924Sarielch                printf("%s checksum is OK\n", $checksum->{'type'});
568c0f6b924Sarielch            }
569c0f6b924Sarielch            else
570c0f6b924Sarielch            {
571c0f6b924Sarielch                unlink($temporary_filename);
572c0f6b924Sarielch                printf("    %s checksum does not match (%s instead of %s)\n",
573c0f6b924Sarielch                       $checksum->{'type'},
574c0f6b924Sarielch                       $file_checksum,
575c0f6b924Sarielch                       $checksum->{'value'});
576c0f6b924Sarielch                return 0;
577c0f6b924Sarielch            }
578c0f6b924Sarielch        }
579c0f6b924Sarielch        else
580c0f6b924Sarielch        {
581c0f6b924Sarielch            # The datafile does not contain a checksum to match against.
582c0f6b924Sarielch            # Display the one that was calculated for the downloaded file so that
583c0f6b924Sarielch            # it can be integrated manually into the data file.
584c0f6b924Sarielch            printf("checksum not given, md5 of file is %s\n", $file_checksum);
585c0f6b924Sarielch            $filename = File::Spec->catfile($ENV{'TARFILE_LOCATION'}, $file_checksum . "-" . $name);
586c0f6b924Sarielch        }
587c0f6b924Sarielch
588c0f6b924Sarielch        rename($temporary_filename, $filename) || die "can not rename $temporary_filename to $filename";
589c0f6b924Sarielch        return 1;
59037e6b05aSAndre Fischer    }
591fb2ebae7Sdamjan    else
592fb2ebae7Sdamjan    {
593c0f6b924Sarielch        unlink($temporary_filename);
594c0f6b924Sarielch        print "    download failed\n";
595c0f6b924Sarielch        return 0;
596fb2ebae7Sdamjan    }
59737e6b05aSAndre Fischer}
59837e6b05aSAndre Fischer
59937e6b05aSAndre Fischer
60037e6b05aSAndre Fischer
60137e6b05aSAndre Fischer
60237e6b05aSAndre Fischer=head3 CheckDownloadDestination ()
60337e6b05aSAndre Fischer
604*e0f16157Smseidel    Make sure that the download destination $TARFILE_LOCATION does exist. If
60537e6b05aSAndre Fischer    not, then the directory is created.
6066a22bca6SAndre Fischer
60737e6b05aSAndre Fischer=cut
60837e6b05aSAndre Fischersub CheckDownloadDestination ()
60937e6b05aSAndre Fischer{
61037e6b05aSAndre Fischer    my $destination = $ENV{'TARFILE_LOCATION'};
61137e6b05aSAndre Fischer    die "ERROR: no destination defined! please set TARFILE_LOCATION!" if ($destination eq "");
61237e6b05aSAndre Fischer
61337e6b05aSAndre Fischer    if ( ! -d $destination)
61437e6b05aSAndre Fischer    {
61537e6b05aSAndre Fischer        File::Path::make_path($destination);
616*e0f16157Smseidel        die "ERROR: can't create \$TARFILE_LOCATION" if ! -d $destination;
61737e6b05aSAndre Fischer    }
61837e6b05aSAndre Fischer}
61937e6b05aSAndre Fischer
62037e6b05aSAndre Fischer
62137e6b05aSAndre Fischer
62237e6b05aSAndre Fischer
62337e6b05aSAndre Fischer=head3 ProvideSpecialTarball ($url,$name,$name_converter)
62437e6b05aSAndre Fischer
625*e0f16157Smseidel    A few tarballs need special handling. That is done here.
6266a22bca6SAndre Fischer
62737e6b05aSAndre Fischer=cut
62837e6b05aSAndre Fischersub ProvideSpecialTarball ($$$)
62937e6b05aSAndre Fischer{
63037e6b05aSAndre Fischer    my $url = shift;
63137e6b05aSAndre Fischer    my $name = shift;
63237e6b05aSAndre Fischer    my $name_converter = shift;
63337e6b05aSAndre Fischer
63437e6b05aSAndre Fischer    return unless defined $url && $url ne "";
63537e6b05aSAndre Fischer
63637e6b05aSAndre Fischer    # See if we can find the executable.
637*e0f16157Smseidel    my ($SOLARENV,$OUTPATH,$EXEEXT) = ($ENV{'SOLARENV'},$ENV{'OUTPATH'},$ENV{'EXEEXT'});
63837e6b05aSAndre Fischer    $SOLARENV = "" unless defined $SOLARENV;
63937e6b05aSAndre Fischer    $OUTPATH = "" unless defined $OUTPATH;
64037e6b05aSAndre Fischer    $EXEEXT = "" unless defined $EXEEXT;
64137e6b05aSAndre Fischer    if (-x File::Spec->catfile($SOLARENV, $OUTPATH, "bin", $name.$EXEEXT))
64237e6b05aSAndre Fischer    {
64337e6b05aSAndre Fischer        print "found $name executable\n";
64437e6b05aSAndre Fischer        return;
64537e6b05aSAndre Fischer    }
64637e6b05aSAndre Fischer
64737e6b05aSAndre Fischer    # Download the source from the URL.
64837e6b05aSAndre Fischer    my $basename = basename(URI->new($url)->path());
64937e6b05aSAndre Fischer    die unless defined $basename;
65037e6b05aSAndre Fischer
65137e6b05aSAndre Fischer    if (defined $name_converter)
65237e6b05aSAndre Fischer    {
65337e6b05aSAndre Fischer        $basename = &{$name_converter}($basename);
65437e6b05aSAndre Fischer    }
6556a22bca6SAndre Fischer
65637e6b05aSAndre Fischer    # Has the source tar ball already been downloaded?
65737e6b05aSAndre Fischer    my @candidates = glob(File::Spec->catfile($ENV{'TARFILE_LOCATION'}, "*-" . $basename));
65837e6b05aSAndre Fischer    if (scalar @candidates > 0)
65937e6b05aSAndre Fischer    {
66037e6b05aSAndre Fischer        # Yes.
66137e6b05aSAndre Fischer        print "$basename exists\n";
66237e6b05aSAndre Fischer        return;
66337e6b05aSAndre Fischer    }
66437e6b05aSAndre Fischer    else
66537e6b05aSAndre Fischer    {
66637e6b05aSAndre Fischer        # No, download it.
66737e6b05aSAndre Fischer        print "downloading $basename\n";
66837e6b05aSAndre Fischer        DownloadFile($basename, $url, undef);
66937e6b05aSAndre Fischer    }
67037e6b05aSAndre Fischer}
67137e6b05aSAndre Fischer
67237e6b05aSAndre Fischer
67337e6b05aSAndre Fischer
67437e6b05aSAndre Fischer
67537e6b05aSAndre Fischer
67637e6b05aSAndre Fischer# The main() functionality.
67737e6b05aSAndre Fischer
67837e6b05aSAndre Fischerdie "usage: $0 <data-file-name>" if scalar @ARGV != 1;
67937e6b05aSAndre Fischermy $data_file = $ARGV[0];
68037e6b05aSAndre FischerCheckDownloadDestination();
68137e6b05aSAndre FischerProcessDataFile($data_file);
68237e6b05aSAndre FischerProvideSpecialTarball($ENV{'DMAKE_URL'}, "dmake", undef);
68337e6b05aSAndre FischerProvideSpecialTarball(
68437e6b05aSAndre Fischer    $ENV{'EPM_URL'},
68537e6b05aSAndre Fischer    "epm",
68637e6b05aSAndre Fischer    sub{$_[0]=~s/-source//; return $_[0]});
687