1#!/usr/bin/env bash 2#************************************************************** 3# 4# Licensed to the Apache Software Foundation (ASF) under one 5# or more contributor license agreements. See the NOTICE file 6# distributed with this work for additional information 7# regarding copyright ownership. The ASF licenses this file 8# to you under the Apache License, Version 2.0 (the 9# "License"); you may not use this file except in compliance 10# with the License. You may obtain a copy of the License at 11# 12# http://www.apache.org/licenses/LICENSE-2.0 13# 14# Unless required by applicable law or agreed to in writing, 15# software distributed under the License is distributed on an 16# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17# KIND, either express or implied. See the License for the 18# specific language governing permissions and limitations 19# under the License. 20# 21#************************************************************** 22 23file_list_name=$1 24 25if [ -z "$TARFILE_LOCATION" ]; then 26 echo "ERROR: no destination defined! please set TARFILE_LOCATION!" 27 exit 28fi 29 30if [ ! -d "$TARFILE_LOCATION" ]; then 31 mkdir $TARFILE_LOCATION 32fi 33if [ ! -d "$TARFILE_LOCATION" ]; then 34 echo "ERROR: can't create" 35 exit 36fi 37 38if [ -z "$1" ]; then 39 echo "ERROR: parameter missing!" 40 echo "usage: $0 <fetch list>" 41 echo "first line must define the base url." 42 exit 43fi 44 45# Downloader method selection 46fetch_bin= 47fetch_args= 48 49#Look for FreeBSD's fetch(1) first 50if [ -x /usr/bin/fetch ]; then 51 fetch_bin=/usr/bin/fetch 52 fetch_args="-Fpr" 53 echo found FreeBSD fetch: $fetch_bin 54 break 1 55else 56 for wg in wget /usr/bin/wget /usr/local/bin/wget /usr/sfw/bin/wget /opt/sfw/bin/wget /opt/local/bin/wget; do 57 eval "$wg --version" > /dev/null 2>&1 58 ret=$? 59 if [ $ret -eq 0 ]; then 60 fetch_bin=$wg 61 fetch_args="-nv -N" 62 echo found wget at `which $fetch_bin` 63 break 2 64 fi 65 done 66 if [ -z "$fetch_bin" ]; then 67 for c in curl /usr/bin/curl /usr/local/bin/curl /usr/sfw/bin/curl /opt/sfw/bin/curl /opt/local/bin/curl; do 68 # mac curl returns "2" on --version 69 # eval "$i --version" > /dev/null 2>&1 70 # ret=$? 71 # if [ $ret -eq 0 ]; then 72 if [ -x $c ]; then 73 fetch_bin=$c 74 fetch_args="$file_date_check -O" 75 echo found curl at `which $fetch_bin` 76 break 2 77 fi 78 done 79 fi 80 if [ -z "$fetch_bin" ]; then 81 echo "ERROR: neither wget nor curl found!" 82 exit 83 fi 84fi 85 86#Checksummer selection 87md5sum= 88 89for i in md5 md5sum /usr/local/bin/md5sum gmd5sum /usr/sfw/bin/md5sum /opt/sfw/bin/gmd5sum /opt/local/bin/md5sum; do 90 if [ "$i" = "md5" ]; then 91 eval "$i -x" > /dev/null 2>&1 92 else 93 eval "$i --version" > /dev/null 2>&1 94 fi 95 ret=$? 96 if [ $ret -eq 0 ]; then 97 md5sum=$i 98 echo found md5sum at `which $md5sum` 99 break 2 100 fi 101done 102 103if [ "$md5sum" = "md5" ]; then 104 md5special=-r 105fi 106 107if [ -z "$md5sum" ]; then 108 echo "Warning: no md5sum: found!" 109fi 110 111start_dir=`pwd` 112logfile=$TARFILE_LOCATION/fetch.log 113date >> $logfile 114 115# Create and go to a temporary directory under the tar file destination. 116mkdir -p $TARFILE_LOCATION/tmp 117cd $TARFILE_LOCATION/tmp 118 119 120basename () 121{ 122 echo $1 | sed "s/^\(.*\/\)//" 123} 124 125 126# 127# Download a file from a URL and add its md5 checksum to its name. 128# 129download () 130{ 131 local URL=$1 132 133 if [ -n "$URL" ]; then 134 local basename=$(basename $URL) 135 local candidate=$(find "$TARFILE_LOCATION" -type f -name "*-$basename") 136 if [ -n "$candidate" ]; then 137 echo "$basename is already present ($candidate)" 138 else 139 echo fetching $basename 140 $fetch_bin $fetch_args $URL 2>&1 | tee -a $logfile 141 142 if [ $? -ne 0 ]; then 143 echo "download failed" 144 mv $basename ${basename}_broken 145 failed="$failed $i" 146 elif [ -f "$basename" -a -n "$md5sum" ]; then 147 local sum=`$md5sum $md5special $basename | sed "s/ .*//"` 148 mv $basename "$TARFILE_LOCATION/$sum-$basename" 149 echo "added md5 sum $sum" 150 fi 151 fi 152 fi 153} 154 155# 156# Download a file from a URL and check its md5 sum to the one that is part of its name. 157# 158download_and_check () 159{ 160 local URL=$1 161 162 if [ -n "$URL" ]; then 163 local basename=$(basename $URL) 164 if [ -f "$TARFILE_LOCATION/$basename" ]; then 165 echo "$basename is already present" 166 else 167 echo "fetching $basename" 168 $fetch_bin $fetch_args $URL 2>&1 | tee -a $logfile 169 170 if [ $? -ne 0 ]; then 171 echo "download failed" 172 mv $basename ${basename}_broken 173 failed="$failed $i" 174 elif [ -f "$basename" -a -n "$md5sum" ]; then 175 local sum=`$md5sum $md5special $basename | sed "s/ .*//"` 176 local sum_in_name=`echo $basename | sed "s/-.*//"` 177 if [ "$sum" != "$sum_in_name" ]; then 178 echo checksum failure for $basename 2>&1 | tee -a $logfile 179 failed="$failed $basename" 180 mv $basename ${basename}_broken 181 fi 182 mv $basename "$TARFILE_LOCATION/$basename" 183 fi 184 fi 185 fi 186} 187 188echo "downloading tar balls to $TARFILE_LOCATION" 189 190while read line ; do 191 # Remove leading and trailing space and comments 192 line=`echo $line | sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/[[:space:]]*#.*$//'` 193 case $line in 194 # Ignore empty lines. 195 '') 196 ;; 197 198 # When a URL ends in a / then it is taken as a partial URL 199 # to which the following lines will be appended. 200 ftp:\/\/*\/ | http:\/\/*\/) 201 UrlHead=$line 202 echo $UrlHead 203 ;; 204 205 # A full URL represents a single file which is downloaded. 206 ftp:\/\/* | http:\/\/*) 207 download $line 208 ;; 209 210 # If the line starts with the name of an environment variable than the file is 211 # downloaded only when the variable evaluates to YES. 212 [A-Z0-9_]*:*) 213 prefix=`echo $line | sed 's/:.*$//'` 214 if [ -n "$prefix" ]; then 215 eval value=\$$prefix 216 if [ "x$value" = "xYES" ]; then 217 line=`echo $line | sed 's/^.*://'` 218 download_and_check $UrlHead$line 219 fi 220 fi 221 ;; 222 223 # Any other line is interpreted as the second part of a partial URL. 224 # It is appended to UrlHead and then downloaded. 225 *) 226 download_and_check $UrlHead$line 227 ;; 228 esac 229done < "$file_list_name" 230 231 232# Special handling of dmake 233if [ -n "$DMAKE_URL" -a ! -x "$SOLARENV/$OUTPATH/bin/dmake$EXEEXT" ]; then 234 download $DMAKE_URL 235fi 236 237# Special handling of epm-3.7 238# Basically just a download of the epm archive. 239# When its name contains "-source" than that part is removed. 240epm_archive_tail=`echo $(basename $EPM_URL) | sed 's/-source//'` 241epm_archive_name=$(find "$TARFILE_LOCATION" -type f -name "*-$epm_archive_tail") 242if [ -n "$EPM_URL" -a ! -x "$SOLARENV/$OUTPATH/bin/epm$EXEEXT" -a -z "$epm_archive_name" ]; then 243 download $EPM_URL 244 archive_name=$(find "$TARFILE_LOCATION" -type f -name "*-epm-3.7-source*") 245 if [ -n "$archive_name" ]; then 246 epm_archive_name=`echo $archive_name | sed 's/-source//'` 247 mv "$archive_name" "$epm_archive_name" 248 fi 249fi 250 251if [ ! -z "$failed" ]; then 252 echo 253 echo ERROR: failed on: 254 for i in $failed ; do 255 echo $i 256 done 257 exit 1 258fi 259 260