1*b1cdbd2cSJim Jagielski#!/usr/bin/env bash 2*b1cdbd2cSJim Jagielski#************************************************************** 3*b1cdbd2cSJim Jagielski# 4*b1cdbd2cSJim Jagielski# Licensed to the Apache Software Foundation (ASF) under one 5*b1cdbd2cSJim Jagielski# or more contributor license agreements. See the NOTICE file 6*b1cdbd2cSJim Jagielski# distributed with this work for additional information 7*b1cdbd2cSJim Jagielski# regarding copyright ownership. The ASF licenses this file 8*b1cdbd2cSJim Jagielski# to you under the Apache License, Version 2.0 (the 9*b1cdbd2cSJim Jagielski# "License"); you may not use this file except in compliance 10*b1cdbd2cSJim Jagielski# with the License. You may obtain a copy of the License at 11*b1cdbd2cSJim Jagielski# 12*b1cdbd2cSJim Jagielski# http://www.apache.org/licenses/LICENSE-2.0 13*b1cdbd2cSJim Jagielski# 14*b1cdbd2cSJim Jagielski# Unless required by applicable law or agreed to in writing, 15*b1cdbd2cSJim Jagielski# software distributed under the License is distributed on an 16*b1cdbd2cSJim Jagielski# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17*b1cdbd2cSJim Jagielski# KIND, either express or implied. See the License for the 18*b1cdbd2cSJim Jagielski# specific language governing permissions and limitations 19*b1cdbd2cSJim Jagielski# under the License. 20*b1cdbd2cSJim Jagielski# 21*b1cdbd2cSJim Jagielski#************************************************************** 22*b1cdbd2cSJim Jagielski 23*b1cdbd2cSJim Jagielskifile_list_name=$1 24*b1cdbd2cSJim Jagielski 25*b1cdbd2cSJim Jagielskiif [ -z "$TARFILE_LOCATION" ]; then 26*b1cdbd2cSJim Jagielski echo "ERROR: no destination defined! please set TARFILE_LOCATION!" 27*b1cdbd2cSJim Jagielski exit 28*b1cdbd2cSJim Jagielskifi 29*b1cdbd2cSJim Jagielski 30*b1cdbd2cSJim Jagielskiif [ ! -d "$TARFILE_LOCATION" ]; then 31*b1cdbd2cSJim Jagielski mkdir $TARFILE_LOCATION 32*b1cdbd2cSJim Jagielskifi 33*b1cdbd2cSJim Jagielskiif [ ! -d "$TARFILE_LOCATION" ]; then 34*b1cdbd2cSJim Jagielski echo "ERROR: can't create" 35*b1cdbd2cSJim Jagielski exit 36*b1cdbd2cSJim Jagielskifi 37*b1cdbd2cSJim Jagielski 38*b1cdbd2cSJim Jagielskiif [ -z "$1" ]; then 39*b1cdbd2cSJim Jagielski echo "ERROR: parameter missing!" 40*b1cdbd2cSJim Jagielski echo "usage: $0 <fetch list>" 41*b1cdbd2cSJim Jagielski echo "first line must define the base url." 42*b1cdbd2cSJim Jagielski exit 43*b1cdbd2cSJim Jagielskifi 44*b1cdbd2cSJim Jagielski 45*b1cdbd2cSJim Jagielski# Downloader method selection 46*b1cdbd2cSJim Jagielskifetch_bin= 47*b1cdbd2cSJim Jagielskifetch_args= 48*b1cdbd2cSJim Jagielski 49*b1cdbd2cSJim Jagielski#Look for FreeBSD's fetch(1) first 50*b1cdbd2cSJim Jagielskiif [ -x /usr/bin/fetch ]; then 51*b1cdbd2cSJim Jagielski fetch_bin=/usr/bin/fetch 52*b1cdbd2cSJim Jagielski fetch_args="-Fpr" 53*b1cdbd2cSJim Jagielski echo found FreeBSD fetch: $fetch_bin 54*b1cdbd2cSJim Jagielskielse 55*b1cdbd2cSJim Jagielski for wg in wget /usr/bin/wget /usr/local/bin/wget /usr/sfw/bin/wget /opt/sfw/bin/wget /opt/local/bin/wget; do 56*b1cdbd2cSJim Jagielski eval "$wg --version" > /dev/null 2>&1 57*b1cdbd2cSJim Jagielski ret=$? 58*b1cdbd2cSJim Jagielski if [ $ret -eq 0 ]; then 59*b1cdbd2cSJim Jagielski fetch_bin=$wg 60*b1cdbd2cSJim Jagielski fetch_args="-nv -N" 61*b1cdbd2cSJim Jagielski echo found wget at `which $fetch_bin` 62*b1cdbd2cSJim Jagielski break 2 63*b1cdbd2cSJim Jagielski fi 64*b1cdbd2cSJim Jagielski done 65*b1cdbd2cSJim Jagielski if [ -z "$fetch_bin" ]; then 66*b1cdbd2cSJim Jagielski for c in curl /usr/bin/curl /usr/local/bin/curl /usr/sfw/bin/curl /opt/sfw/bin/curl /opt/local/bin/curl; do 67*b1cdbd2cSJim Jagielski # mac curl returns "2" on --version 68*b1cdbd2cSJim Jagielski # eval "$i --version" > /dev/null 2>&1 69*b1cdbd2cSJim Jagielski # ret=$? 70*b1cdbd2cSJim Jagielski # if [ $ret -eq 0 ]; then 71*b1cdbd2cSJim Jagielski if [ -x $c ]; then 72*b1cdbd2cSJim Jagielski fetch_bin=$c 73*b1cdbd2cSJim Jagielski fetch_args="$file_date_check -O" 74*b1cdbd2cSJim Jagielski echo found curl at `which $fetch_bin` 75*b1cdbd2cSJim Jagielski break 2 76*b1cdbd2cSJim Jagielski fi 77*b1cdbd2cSJim Jagielski done 78*b1cdbd2cSJim Jagielski fi 79*b1cdbd2cSJim Jagielski if [ -z "$fetch_bin" ]; then 80*b1cdbd2cSJim Jagielski echo "ERROR: neither wget nor curl found!" 81*b1cdbd2cSJim Jagielski exit 82*b1cdbd2cSJim Jagielski fi 83*b1cdbd2cSJim Jagielskifi 84*b1cdbd2cSJim Jagielski 85*b1cdbd2cSJim Jagielski#Checksummer selection 86*b1cdbd2cSJim Jagielskimd5sum= 87*b1cdbd2cSJim Jagielski 88*b1cdbd2cSJim Jagielskifor i in md5 md5sum /usr/local/bin/md5sum gmd5sum /usr/sfw/bin/md5sum /opt/sfw/bin/gmd5sum /opt/local/bin/md5sum; do 89*b1cdbd2cSJim Jagielski if [ "$i" = "md5" ]; then 90*b1cdbd2cSJim Jagielski eval "$i -x" > /dev/null 2>&1 91*b1cdbd2cSJim Jagielski else 92*b1cdbd2cSJim Jagielski eval "$i --version" > /dev/null 2>&1 93*b1cdbd2cSJim Jagielski fi 94*b1cdbd2cSJim Jagielski ret=$? 95*b1cdbd2cSJim Jagielski if [ $ret -eq 0 ]; then 96*b1cdbd2cSJim Jagielski md5sum=$i 97*b1cdbd2cSJim Jagielski echo found md5sum at `which $md5sum` 98*b1cdbd2cSJim Jagielski break 2 99*b1cdbd2cSJim Jagielski fi 100*b1cdbd2cSJim Jagielskidone 101*b1cdbd2cSJim Jagielski 102*b1cdbd2cSJim Jagielskiif [ "$md5sum" = "md5" ]; then 103*b1cdbd2cSJim Jagielski md5special=-r 104*b1cdbd2cSJim Jagielskifi 105*b1cdbd2cSJim Jagielski 106*b1cdbd2cSJim Jagielskiif [ -z "$md5sum" ]; then 107*b1cdbd2cSJim Jagielski echo "Warning: no md5sum: found!" 108*b1cdbd2cSJim Jagielskifi 109*b1cdbd2cSJim Jagielski 110*b1cdbd2cSJim Jagielskistart_dir=`pwd` 111*b1cdbd2cSJim Jagielskilogfile=$TARFILE_LOCATION/fetch.log 112*b1cdbd2cSJim Jagielskidate >> $logfile 113*b1cdbd2cSJim Jagielski 114*b1cdbd2cSJim Jagielski# Create and go to a temporary directory under the tar file destination. 115*b1cdbd2cSJim Jagielskimkdir -p $TARFILE_LOCATION/tmp 116*b1cdbd2cSJim Jagielskicd $TARFILE_LOCATION/tmp 117*b1cdbd2cSJim Jagielski 118*b1cdbd2cSJim Jagielski 119*b1cdbd2cSJim Jagielskibasename () 120*b1cdbd2cSJim Jagielski{ 121*b1cdbd2cSJim Jagielski echo $1 | sed "s/^\(.*\/\)//" 122*b1cdbd2cSJim Jagielski} 123*b1cdbd2cSJim Jagielski 124*b1cdbd2cSJim Jagielski 125*b1cdbd2cSJim Jagielski# 126*b1cdbd2cSJim Jagielski# Download a file from a URL and add its md5 checksum to its name. 127*b1cdbd2cSJim Jagielski# 128*b1cdbd2cSJim Jagielskidownload () 129*b1cdbd2cSJim Jagielski{ 130*b1cdbd2cSJim Jagielski local URL=$1 131*b1cdbd2cSJim Jagielski 132*b1cdbd2cSJim Jagielski if [ -n "$URL" ]; then 133*b1cdbd2cSJim Jagielski local basename=$(basename $URL) 134*b1cdbd2cSJim Jagielski local candidate=$(find "$TARFILE_LOCATION" -type f -name "*-$basename") 135*b1cdbd2cSJim Jagielski if [ -n "$candidate" ]; then 136*b1cdbd2cSJim Jagielski echo "$basename is already present ($candidate)" 137*b1cdbd2cSJim Jagielski else 138*b1cdbd2cSJim Jagielski echo fetching $basename 139*b1cdbd2cSJim Jagielski $fetch_bin $fetch_args $URL 2>&1 | tee -a $logfile 140*b1cdbd2cSJim Jagielski 141*b1cdbd2cSJim Jagielski if [ $? -ne 0 ]; then 142*b1cdbd2cSJim Jagielski echo "download failed" 143*b1cdbd2cSJim Jagielski mv $basename ${basename}_broken 144*b1cdbd2cSJim Jagielski failed="$failed $i" 145*b1cdbd2cSJim Jagielski elif [ -f "$basename" -a -n "$md5sum" ]; then 146*b1cdbd2cSJim Jagielski local sum=`$md5sum $md5special $basename | sed "s/ .*//"` 147*b1cdbd2cSJim Jagielski mv $basename "$TARFILE_LOCATION/$sum-$basename" 148*b1cdbd2cSJim Jagielski echo "added md5 sum $sum" 149*b1cdbd2cSJim Jagielski fi 150*b1cdbd2cSJim Jagielski fi 151*b1cdbd2cSJim Jagielski fi 152*b1cdbd2cSJim Jagielski} 153*b1cdbd2cSJim Jagielski 154*b1cdbd2cSJim Jagielski# 155*b1cdbd2cSJim Jagielski# Download a file from a URL and check its md5 sum to the one that is part of its name. 156*b1cdbd2cSJim Jagielski# 157*b1cdbd2cSJim Jagielskidownload_and_check () 158*b1cdbd2cSJim Jagielski{ 159*b1cdbd2cSJim Jagielski local URL=$1 160*b1cdbd2cSJim Jagielski 161*b1cdbd2cSJim Jagielski if [ -n "$URL" ]; then 162*b1cdbd2cSJim Jagielski local basename=$(basename $URL) 163*b1cdbd2cSJim Jagielski if [ -f "$TARFILE_LOCATION/$basename" ]; then 164*b1cdbd2cSJim Jagielski echo "$basename is already present" 165*b1cdbd2cSJim Jagielski else 166*b1cdbd2cSJim Jagielski echo "fetching $basename" 167*b1cdbd2cSJim Jagielski $fetch_bin $fetch_args $URL 2>&1 | tee -a $logfile 168*b1cdbd2cSJim Jagielski 169*b1cdbd2cSJim Jagielski if [ $? -ne 0 ]; then 170*b1cdbd2cSJim Jagielski echo "download failed" 171*b1cdbd2cSJim Jagielski mv $basename ${basename}_broken 172*b1cdbd2cSJim Jagielski failed="$failed $i" 173*b1cdbd2cSJim Jagielski elif [ -f "$basename" -a -n "$md5sum" ]; then 174*b1cdbd2cSJim Jagielski local sum=`$md5sum $md5special $basename | sed "s/ .*//"` 175*b1cdbd2cSJim Jagielski local sum_in_name=`echo $basename | sed "s/-.*//"` 176*b1cdbd2cSJim Jagielski if [ "$sum" != "$sum_in_name" ]; then 177*b1cdbd2cSJim Jagielski echo checksum failure for $basename 2>&1 | tee -a $logfile 178*b1cdbd2cSJim Jagielski failed="$failed $basename" 179*b1cdbd2cSJim Jagielski mv $basename ${basename}_broken 180*b1cdbd2cSJim Jagielski fi 181*b1cdbd2cSJim Jagielski mv $basename "$TARFILE_LOCATION/$basename" 182*b1cdbd2cSJim Jagielski fi 183*b1cdbd2cSJim Jagielski fi 184*b1cdbd2cSJim Jagielski fi 185*b1cdbd2cSJim Jagielski} 186*b1cdbd2cSJim Jagielski 187*b1cdbd2cSJim Jagielskiecho "downloading tar balls to $TARFILE_LOCATION" 188*b1cdbd2cSJim Jagielski 189*b1cdbd2cSJim Jagielskiwhile read line ; do 190*b1cdbd2cSJim Jagielski # Remove leading and trailing space and comments 191*b1cdbd2cSJim Jagielski line=`echo $line | sed 's/^\s*//;s/\s*$//;s/\s*#.*$//'` 192*b1cdbd2cSJim Jagielski case $line in 193*b1cdbd2cSJim Jagielski # Ignore empty lines. 194*b1cdbd2cSJim Jagielski '') 195*b1cdbd2cSJim Jagielski ;; 196*b1cdbd2cSJim Jagielski 197*b1cdbd2cSJim Jagielski # When a URL ends in a / then it is taken as a partial URL 198*b1cdbd2cSJim Jagielski # to which the following lines will be appended. 199*b1cdbd2cSJim Jagielski ftp:\/\/*\/ | http:\/\/*\/) 200*b1cdbd2cSJim Jagielski UrlHead=$line 201*b1cdbd2cSJim Jagielski echo $UrlHead 202*b1cdbd2cSJim Jagielski ;; 203*b1cdbd2cSJim Jagielski 204*b1cdbd2cSJim Jagielski # A full URL represents a single file which is downloaded. 205*b1cdbd2cSJim Jagielski ftp:\/\/* | http:\/\/*) 206*b1cdbd2cSJim Jagielski download $line 207*b1cdbd2cSJim Jagielski ;; 208*b1cdbd2cSJim Jagielski 209*b1cdbd2cSJim Jagielski # If the line starts with the name of an environment variable than the file is 210*b1cdbd2cSJim Jagielski # downloaded only when the variable evaluates to YES. 211*b1cdbd2cSJim Jagielski [A-Z0-9_]*:*) 212*b1cdbd2cSJim Jagielski prefix=`echo $line | sed 's/:.*$//'` 213*b1cdbd2cSJim Jagielski if [ -n "$prefix" ]; then 214*b1cdbd2cSJim Jagielski eval value=\$$prefix 215*b1cdbd2cSJim Jagielski if [ "x$value" = "xYES" ]; then 216*b1cdbd2cSJim Jagielski line=`echo $line | sed 's/^.*://'` 217*b1cdbd2cSJim Jagielski download_and_check $UrlHead$line 218*b1cdbd2cSJim Jagielski fi 219*b1cdbd2cSJim Jagielski fi 220*b1cdbd2cSJim Jagielski ;; 221*b1cdbd2cSJim Jagielski 222*b1cdbd2cSJim Jagielski # Any other line is interpreted as the second part of a partial URL. 223*b1cdbd2cSJim Jagielski # It is appended to UrlHead and then downloaded. 224*b1cdbd2cSJim Jagielski *) 225*b1cdbd2cSJim Jagielski download_and_check $UrlHead$line 226*b1cdbd2cSJim Jagielski ;; 227*b1cdbd2cSJim Jagielski esac 228*b1cdbd2cSJim Jagielskidone < "$file_list_name" 229*b1cdbd2cSJim Jagielski 230*b1cdbd2cSJim Jagielski 231*b1cdbd2cSJim Jagielski# Special handling of dmake 232*b1cdbd2cSJim Jagielskiif [ -n "$DMAKE_URL" -a ! -x "$SOLARENV/$OUTPATH/bin/dmake$EXEEXT" ]; then 233*b1cdbd2cSJim Jagielski download $DMAKE_URL 234*b1cdbd2cSJim Jagielskifi 235*b1cdbd2cSJim Jagielski 236*b1cdbd2cSJim Jagielski# Special handling of epm-3.7 237*b1cdbd2cSJim Jagielski# Basically just a download of the epm archive. 238*b1cdbd2cSJim Jagielski# When its name contains "-source" than that part is removed. 239*b1cdbd2cSJim Jagielskiepm_archive_tail=`echo $(basename $EPM_URL) | sed 's/-source//'` 240*b1cdbd2cSJim Jagielskiepm_archive_name=$(find "$TARFILE_LOCATION" -type f -name "*-$epm_archive_tail") 241*b1cdbd2cSJim Jagielskiif [ -n "$EPM_URL" -a ! -x "$SOLARENV/$OUTPATH/bin/epm$EXEEXT" -a -z "$epm_archive_name" ]; then 242*b1cdbd2cSJim Jagielski download $EPM_URL 243*b1cdbd2cSJim Jagielski archive_name=$(find "$TARFILE_LOCATION" -type f -name "*-epm-3.7-source*") 244*b1cdbd2cSJim Jagielski if [ -n "$archive_name" ]; then 245*b1cdbd2cSJim Jagielski epm_archive_name=`echo $archive_name | sed 's/-source//'` 246*b1cdbd2cSJim Jagielski mv "$archive_name" "$epm_archive_name" 247*b1cdbd2cSJim Jagielski fi 248*b1cdbd2cSJim Jagielskifi 249*b1cdbd2cSJim Jagielski 250*b1cdbd2cSJim Jagielskiif [ ! -z "$failed" ]; then 251*b1cdbd2cSJim Jagielski echo 252*b1cdbd2cSJim Jagielski echo ERROR: failed on: 253*b1cdbd2cSJim Jagielski for i in $failed ; do 254*b1cdbd2cSJim Jagielski echo $i 255*b1cdbd2cSJim Jagielski done 256*b1cdbd2cSJim Jagielski exit 1 257*b1cdbd2cSJim Jagielskifi 258*b1cdbd2cSJim Jagielski 259