#!/usr/bin/perl # use instead /usr/bin/env perl ? =head1 NAME installargos.pl - install A Replicable Genome infOrmation System (ARGOS) =head1 SYNOPSIS install script for ARGOS, A Replicable Genome infOrmation System. We would rather use rpm, pacman or other package install manager, but available ones are not quite right. This script now lives at rsync://eugenes.org/argos/install/installargos.pl Server packages are (mostly) at rsync://eugenes.org/argos =head1 DESCRIPTION Summary of steps to installation of a FlyBase-NG server 0. Fetch the install script from command line with rsync rsync://eugenes.org/argos/install/installargos.pl . 1. Run installargos.pl -root=/usr/local/argos -install to create root folder and fetch the installation package (location for -root= is your choice; change below to match) 2. Edit /usr/local/argos/install/install.conf.local to set configuration. Change package set, paths and ports as desired in this install.conf. 3. Run /usr/local/argos/install/installargos.pl -install to add the full set of packages. Packages selected from packages.conf will by copied from servers. 4. Run /usr/local/argos/install/run-apache to start servers 5+. Run /usr/local/argos/install/installargos.pl -update to update server periodically. The -info and -list commands make no changes to disk. All commands are benign if used with -n[orun] option. Some other installargos options: -info == show packages information -list -pack flybase == list all files in package, and required packages, from distribution servers. -install == sets root path; creates ARGOS_ROOT/install/; installs common packages by rsync. -update == update installed packages, for frequent, automatic usage (e.g. cron job) -list -update == list updates that are needed -reconfig == reset root, port, other configurations to install.conf.local values -no -install == view steps without doing them -root=/usr/local/argos == use new root path =head1 AUTHORS Don Gilbert, Nihar Sheth ; june 2003 email: argos@eugenes.org =head1 NOTES sep04: FIXME - messes up when cgi-made install.conf.local is in same folder as perl installargos.pl is bootstrapping in; i.e. Step 3 before Step 1-2. 12feb04 -- changed ROOT to ARGOS_ROOT (argh..) through all config scripts -- renamed major keys in install.conf; install.conf.local's will get out of date! 23oct03 -- must read argos-install package first and update packages.conf if needed -- -clean makes a mess by deleting things other packages re-install -- -reconfig(ure) can mess up apache.conf settings - needs to be smarter -- need more post-processing checks, edits -- should default ARGOS_ROOT be install.conf == /bio/biodb or current user folder? -- want web form to use this; read packages - write install.conf.local -- not here, but need better run- script handling ** dang ! CVS excludes '*.orig' files, which we want in CVS ... change to .default (apache uses) or .dist ? ** 1st package install is a lot of data by individual files over rsync; - improve efficiency & server load by adding tar.gz files of main packages - check & install those 1st then update by rsync? ** need to fix update to ignore or sensibly update local .conf files -info == add test to see if package needs update? =cut ArgosInstaller::main(); #-------------------------------------------------------------- package ArgosInstaller; use strict; use Getopt::Long; use vars qw() ; use vars qw( $configFileOk $debug %OldKeys $doPatchOldKeys %packlist @packlist %origconfig %config %configchange $ARGOS_ROOT $DEFAULT_ROOT $VERSION $NG_CONF $NG_CONF_DEFAULT $NG_CRONTAB $PACKAGE_LIST $CONFIGCHANGE_LIST $DEFAULT_PACKAGE $COMMON_EXCLUDE $LOCAL_EXCLUDE %INSTALL_PACK %DEFAULT_CONF $sufLOCAL $sufDEFAULT $verbose $shellquiet $viewOnly $doinstall $doupdate $dopackinfo $dopacklist $isempty $isfull $dorun $reconfigure $doHelp $beQuiet $cleanupd $userroot $installroot @userpacks @rsyncopts $haversync $skiprequires $program $userexcludefile %runopts $doall $urlhost ); =item -- fix .orig naming problem set fset=`find . -name '*.orig'` foreach f ($fset) set fto=`echo $f | sed -e 's/.orig/.default/'` mv $f $fto end =cut BEGIN { $VERSION = 0.7; $doPatchOldKeys=1; %OldKeys = ( # install.conf changes from Dec2003 -> Feb2004 ROOT => 'ARGOS_ROOT', APACHE_PORT => 'ARGOS_APACHE', # TOMCAT_PORT => '', #? gone PGPORT => 'ARGOS_POSTGRES', FLYBASE_PORT => 'FLYBASE_APACHE', TOMCAT_FLYBASE => 'FLYBASE_TOMCAT', EUGENES_PORT => 'EUGENES_APACHE', TOMCAT_EUGENES => 'EUGENES_TOMCAT', DAPHNIA_PORT => 'DAPHNIA_APACHE', TOMCAT_DAPHNIA => 'DAPHNIA_TOMCAT', CENTAURBASE_PORT => 'CENTAURBASE_APACHE', TOMCAT_CENTAURBASE => 'CENTAURBASE_TOMCAT', ); $NG_CONF = "install.conf.local"; $NG_CONF_DEFAULT = "install.conf"; $NG_CRONTAB = "argos-crontab.txt"; $PACKAGE_LIST = "packages.conf"; # need pack.conf.local sometimes $CONFIGCHANGE_LIST = "reconfigure.conf"; # reconfigure.conf ? $DEFAULT_ROOT = "/bio/argos"; $DEFAULT_PACKAGE = 'argos-install'; $COMMON_EXCLUDE= "install/rsync.exclude"; $LOCAL_EXCLUDE= "install/rsync.exclude.local"; $sufLOCAL= ".local"; $sufDEFAULT= ".default"; # was .orig - cvs excludes %INSTALL_PACK = ( $DEFAULT_PACKAGE => { 'package' => $DEFAULT_PACKAGE, description => 'installation package', url => 'rsync://eugenes.org/argos/install/', localpath => 'install/', }, ); %DEFAULT_CONF = ( ARGOS_ROOT => $DEFAULT_ROOT , PACKAGES => $DEFAULT_PACKAGE, ); $verbose= $shellquiet= 0; ## == echoFlag $viewOnly= 1; $doinstall= 0; $doupdate= 0; $dopackinfo= 0; $dopacklist= 0; $doall= 0; $isempty= 0; $isfull= 0; $debug= 0; $dorun= 1; $reconfigure= 0; $doHelp= 0; $beQuiet= 0; $cleanupd= 0; $userroot= undef; $installroot= undef; @userpacks= (); @rsyncopts= (); $urlhost=''; $skiprequires= 0; $program = $0; %runopts= ( 'all' => \$doall, 'clean' => \$cleanupd, 'debug=s' => \$debug, 'help' => \$doHelp, 'info' => \$dopackinfo, 'inroot=s' => \$installroot, # webroot ?? 'install' => \$doinstall, 'list' => \$dopacklist, 'package=s' => \@userpacks, 'program=s' => \$program, 'quiet' => \$beQuiet, 'reconfigure' => \$reconfigure, 'root=s' => \$userroot, 'rsopts=s' => \@rsyncopts, 'run!' => \$dorun, # trick for -norun == -n == dryrun 'skiprequires' => \$skiprequires, 'update' => \$doupdate, 'urlhost=s' => \$urlhost, 'userexcludefile=s' => \$userexcludefile, 'verbose' => \$verbose, ); } # --- main --- sub main { my $ok= GetOptions(%runopts); getConfigs($ok); if ($beQuiet) {} # what is this for? - install.cgi uses it to init base package elsif ($doHelp) { doHelp( $isempty, $isfull, $haversync); } #exit(0); else { doWork(); } } #---------------- ## NOTE: need to re-read packages if updated .. before finishing rest of update ! sub rereadPackages { %packlist = readPackagesWithLocal("$ARGOS_ROOT/install/$PACKAGE_LIST",\%INSTALL_PACK); warn "rereadPackages($ARGOS_ROOT/install/$PACKAGE_LIST)\n" if $debug; $isempty=1 unless($configFileOk); # @packlist = sort keys %packlist; @packlist= getPackages(@userpacks); } sub getConfigs { my ($ok)= @_; # -- check configs now (using $userroot if given) $haversync=`rsync --version`; @rsyncopts= split(/,/,join(',',@rsyncopts)); my $pg= $program; #print STDERR "pg=$pg\n"; unless($pg) { $pg='./'; } my $instroot= ($pg =~ m,\$\-,) ? './' : `dirname $pg`; chomp($instroot); if ($installroot && $installroot !~ m,^/,) { require File::Spec; $installroot= File::Spec->rel2abs($installroot); } ## if $userroot - need to convert to full path if relative if ($userroot && $userroot !~ m,^/,) { require File::Spec; $userroot= File::Spec->rel2abs($userroot); } $instroot= "$userroot/install" if ($userroot && -d "$userroot/install"); #? need packlist in original order? not hash order? %packlist = readPackagesWithLocal("$instroot/$PACKAGE_LIST",\%INSTALL_PACK); $isempty=1 unless($configFileOk); %origconfig = readConfig("$instroot/$NG_CONF_DEFAULT",\%DEFAULT_CONF); $isempty=1 unless($configFileOk); %config = readConfig("$instroot/$NG_CONF",\%origconfig); patchOldKeys(\%OldKeys, \%config) if ($doPatchOldKeys); #? only for NG_CONF %configchange = readConfig("$instroot/$CONFIGCHANGE_LIST",{}); @packlist = sort keys %packlist; # my $conflist = join(", ", map { "$_=$config{$_}" } sort keys %config ); $ARGOS_ROOT= $userroot || $config{ARGOS_ROOT}; $isfull= (-e "$ARGOS_ROOT/common/servers"); #? $urlhost= $config{URL_HOST} if (!$urlhost && $config{URL_HOST}); $urlhost =~ s/[:,;].*$//; $doinstall= $doupdate= $reconfigure= 0 unless ($haversync =~ /version/); $doHelp= 1 unless($ok && ($doinstall || $dopackinfo || $dopacklist || $doupdate || $reconfigure)); # if($doHelp) { doHelp( $isempty, $isfull, $haversync); exit(0); } } sub patchOldKeys { my ($oldKeys, $config)= @_; foreach my $oldkey (keys %$oldKeys) { my $newkey= $oldKeys->{$oldkey}; if ($config->{$oldkey}) { $config->{$newkey}= $config->{$oldkey}; } } } sub doWork { $viewOnly= !$dorun; $verbose= 1 if ($viewOnly); @packlist= getPackages(@userpacks); if ($dopackinfo) { foreach my $pack (@packlist) { packageInfo($pack); } } elsif ($dopacklist) { my $action=($doall) ? 'LIST_ALL' : ($doupdate) ? 'LIST_UPDATE' :'LIST'; foreach my $pack (@packlist) { listPackage($pack, $action); } ## add list updates == rsopts=-v,-n for updatePackage } elsif ($doinstall) { warn "\n# INSTALL server directories\n" if $verbose; ## this should be safe to rerun ## does only updatePackage + reconfigure when called again # die " Installation exists at $ARGOS_ROOT\n Please remove or use another -root\n" # if (-f "$ARGOS_ROOT/$LOCAL_EXCLUDE"); warn " Installation exists at $ARGOS_ROOT\n Doing -update + -reconfig\n" if (-f "$ARGOS_ROOT/$LOCAL_EXCLUDE"); doShell("mkdir","-p","$ARGOS_ROOT/install") unless(-d "$ARGOS_ROOT/install"); my $OS = findOS(); warn "# OS=$OS\n" if ($verbose); ## check server for systems/$OS .. my @oslist= availableSystems(); my $haveos = grep { $_ eq $OS } @oslist; #my $haveos = grep( /^$OS$/, @oslist); unless( $haveos ) { warn "This '$OS' is not a supported operating system\nAvailable systems are " .join("\n",@oslist)."\n"; die unless($viewOnly); } makeLocalExclude($OS, "$ARGOS_ROOT/$LOCAL_EXCLUDE", \@oslist) unless(-f "$ARGOS_ROOT/$LOCAL_EXCLUDE"); foreach my $pack (@packlist) { updatePackage('INSTALL',$pack); } $doupdate=0; ## need to do these after update/install packages ## each package may have optional post-fetch commands (e.g. rpm) unless(-f "$ARGOS_ROOT/install/$NG_CONF") { doShell("cp","-p","$ARGOS_ROOT/install/$NG_CONF_DEFAULT","$ARGOS_ROOT/install/$NG_CONF"); # update ng_conf for user choices; editConfigFiles needs fixing to write these # also want, here or thru package postinstall, to edit ng_conf FLYBASE_CONF=#FLYBASE_CONF # etc to enable .. my $ed=''; ## change ARGOS_ROOT also for new install in local folder ? if ($userroot) { $config{ARGOS_ROOT}= $userroot; $ed.="s,^ARGOS_ROOT=\\S+,ARGOS_ROOT=$userroot,;"; } if (@userpacks) { my $ps= $config{PACKAGES}= join(',',@userpacks); $ed .= "s,^PACKAGES=\\S+,PACKAGES='$ps',;"; } doShell("perl","-pi","-e","$ed","$ARGOS_ROOT/install/$NG_CONF") if ($ed); } doShell("mkdir","-p","$ARGOS_ROOT/common/") unless(-d "${ARGOS_ROOT}/common/"); doShell("ln", "-s", "./systems/${OS}", "${ARGOS_ROOT}/common/system-local") unless(-e "${ARGOS_ROOT}/common/system-local"); # editConfigFiles(\%configchange); $reconfigure=1; } # die "not installed at $ARGOS_ROOT" unless($viewOnly || -d "$ARGOS_ROOT") ; elsif ( $doupdate ) { warn "\n# UPDATE packages\n" if $verbose; my @inpack = grep(/install/, @packlist); if (@inpack) { foreach my $pack (@inpack) { updatePackage('UPDATE',$pack); } rereadPackages(); } foreach my $pack (@packlist) { updatePackage('UPDATE',$pack); } # need to check files in %configchange and set $reconfigure=1; if $sufDEFAULT files } # may need to run this after any updatePackage if ( $reconfigure ) { warn "\n# RECONFIGURE packages\n" if $verbose; editConfigFiles(\%configchange); } } #-------- subs ----------------------------- # for cgi sub installRoot { return $installroot || $userroot || $config{ARGOS_ROOT}; } sub doHelp { my ($isempty,$isfull, $haversync)= @_; my $confroot= $userroot || $config{ARGOS_ROOT}; my $conflist = join(", ", map { "$_=$config{$_}" } sort keys %config ); my $one= ($isempty) ? "** 1." : " 1."; my $two= ($isempty) ? "** 2." : " 2."; my $tre= ($isempty||$isfull) ? " 3." : "** 3."; print <<"HERE"; Usage: $0 [ options ] -info|install|list|update|reconfigure $one Run $0 -install [-root=$confroot] == bootstrap install $two Edit $ARGOS_ROOT/install/$NG_CONF to set configuration. $tre Run $ARGOS_ROOT/install/install -install == copy server packages Packages selected from $PACKAGE_LIST will by copied from servers. Start web servers with $ARGOS_ROOT/install/run-apache Web notes are at http://eugenes.org/argos/install/ Commands: -info == list package info -install == install new server set -update == update current server set -list == list package files at server -list -update == list updates needed -list -all == list all (like -list, but in parsable format) -reconfig == change to new $NG_CONF configuration Options: -root=/path/to/install == override ARGOS_ROOT=$confroot -package=a,b, ... == use specific packages ('all' for everyone) -rsopts=-v,-n ... == apply rsync flags (-n=show dry-run, -v=verbose) -clean == clean update (delete non-server files; not yet default) -skiprequires == skip required sub-packages of package (update) -userexcludefile == file of rsync excludes, in addition to install/rsync.exclude* -n, -norun == don't do anything except show commands -v, -verbose == show commands -h, -help == this help configuration= $conflist packages= @packlist HERE unless($haversync =~ /version/){ print <<"HERE"; This installer requires rsync ( http://rsync.samba.org/ ) which isn't on path - please install it first. Find compiled programs at http://rsync.samba.org/ftp/rsync/binaries/ ftp://rsync.samba.org/pub/rsync/binaries/ HERE } } sub doShell { my @cmd= @_; my $result= ''; ## too bad cant do `@cmd` ; use system w/ output redir ?? warn join(' ',@cmd)."\n" if ($verbose && !$shellquiet); unless($viewOnly) { $|= 1; ## system writes to stdout - ok my $err= system( @cmd); ## , "> $tmpf" # no good unless cmd is /bin/(c)sh warn "Error=$err\n" if $err; } ## need to check each @_ for unix special chars unless '' or "" quoted # my $cmd = join(" ",@_); # warn "$cmd\n" if $verbose; # $result= `$cmd` unless($viewOnly); warn " =$result=\n" if ($verbose && !$shellquiet && $result); return $result; } sub updatePackage { my ($action,$pack)= @_; my $packh= $packlist{$pack}; unless($packh) { warn "no $pack package\n"; return; } my $to= ${ARGOS_ROOT}."/".$packh->{localpath}; $to .= "/" unless($to =~ m,/$,); warn "\n# $action package=$pack\n" if $verbose; gotoPackage($action,$pack,$to); if ($packh->{postinstall}) { warn "# POSTINSTALL $pack script\n" if $verbose; doShell( split(' ',$packh->{postinstall})); } } sub listPackage { my ($pack, $action)= @_; @rsyncopts=('-n','-v'); #,'--stats' $verbose=1; $shellquiet=1; my $to=""; $action='LIST' unless($action); if ( $action =~ /UPDATE|ALL/i ) { # my $packh= $packlist{$pack}; $to= ${ARGOS_ROOT}."/".$packh->{localpath}; $to .= "/" unless($to =~ m,/$,); } print "\n# $action package=$pack\n" if $verbose; gotoPackage($action,$pack,$to); } sub gotoPackage { my ($action,$pack,$todir)= @_; my $packh= $packlist{$pack}; #my $from = $packh->{url}; ## add support for {'alt-url'} ; maybe named 'url-$althost' ? my $from = $packh->{"url-$urlhost"} || $packh->{url}; my $excludes = $packh->{exclude}; my $instupd= ($action ne 'LIST'); #?? if ($action =~ m/UPDATE/ && $packh->{updateexclude}) { $excludes .= " " if ($excludes); $excludes .= $packh->{updateexclude}; } my $excludelist= getExcludeList( $action, $excludes, $packh->{excludefrom}); # common exfrom files or package-specific? my @parts= (""); if ($packh->{userparts}) { @parts= split( /[,\s]+/, $packh->{userparts} ); } elsif ($packh->{parts}) { @parts= split( /[,\s]+/, $packh->{parts} ); } foreach my $part (@parts) { ##... loop thru and append to $from, $to $part .= "/" if($part && $part !~ m,/$,); my $topart= ($instupd) ? $todir.$part : $todir; ##? check that $from =~ rsync? or need -e ssh and user info my $res= doRsync( $from.$part, $topart, $action, $excludelist); ## ? list $res for list unless $verbose ? } } sub getExcludeList { my ($action, $excludes, $packexfrom)= @_; my @excludes=(); foreach my $ex ($COMMON_EXCLUDE, $LOCAL_EXCLUDE, $userexcludefile, $packexfrom) { next unless $ex; $ex = "${ARGOS_ROOT}/$ex" unless -e $ex; if ($viewOnly || -e $ex) { push @excludes, "--exclude-from=$ex"; if ($action =~ /^LIST_ALL/i) { print "# EXCLUDE-FROM:$ex\n"; } } } if ($excludes) { my @ex= split( /[,\s]+/, $excludes); # this split gets leading ' ' foreach my $e (@ex) { push @excludes, "--exclude=$e" if ($e =~ /\S/); if (($action =~ /^LIST_ALL/i) && ($e =~ /\S/)) { print "# EXCLUDE:$e\n"; } } } return \@excludes; } sub listPackageForCgi { my ($pack,$listupdate)= @_; ## should be same as listPackage() my $ex; my $verbose= 1; my $todir=''; my @rsyncopts=('-n','-v','--stats'); my @cmd = qw(rsync -a); push @cmd, @rsyncopts; my $action='LIST'; #? optionally LIST_UPDATE # $action='LIST_UPDATE' if ($listupdate); # -- gotoPackage my $packh= $packlist{$pack}; ##my $from = $packh->{url}; my $from = $packh->{"url-$urlhost"} || $packh->{url}; my $excludes = $packh->{exclude}; # if ($action =~ /UPDATE/ && $packh->{updateexclude}) { # $excludes .= " " if ($excludes); # $excludes .= $packh->{updateexclude}; # } # if($action =~ /UPDATE/) { # push @cmd, "-u","--modify-window=130"; # push @cmd,"--delete" if ($cleanupd); # } my $excludelist= getExcludeList( $action, $excludes, $packh->{excludefrom}); push @cmd, @$excludelist if (ref($excludes) && @$excludelist); my @parts= (""); if ($packh->{userparts}) { @parts= split( /[,\s]+/, $packh->{userparts} ); } elsif ($packh->{parts}) { @parts= split( /[,\s]+/, $packh->{parts} ); } foreach my $part (@parts) { $part .= "/" if($part && $part !~ m,/$,); # -- doRsync my ($frompt) = ($from.$part); ## need a local todir to check updates ##? my $topart= ($instupd) ? $todir.$part : $todir; print "# $action $frompt\n"; unless($frompt) { print "# Error - missing source\n"; next; } my @fcmd= @cmd; push @fcmd, $frompt; # print join(' ',@fcmd)."\n" if $verbose; # my $cmd= join(' ',@cmd,$frompt); #? safe #`$cmd`; my $err= system( @fcmd); print "Error=$err\n" if $err; print "\n"; } } sub packageInfo { my ($pack)= @_; my $packh= $packlist{$pack}; print "package: $pack\n"; foreach my $k (sort keys %$packh) { next if ($k eq 'package'); my $v= $packh->{$k}; $v= ${ARGOS_ROOT}."/".$v if ($k eq 'localpath'); print "$k: $v\n"; } print "\n"; } sub getPackages { my @pl= @_; if (@pl) { # user choices @pl = split(/,/,join(',',@pl)); @pl = sort keys %packlist if ($pl[0] eq 'all'); } elsif ($config{PACKAGES}) { @pl = split(/[,\s]+/,$config{PACKAGES}); @pl = sort keys %packlist if ($pl[0] eq 'all'); } # else what ? foreach my $pn (@pl) { my $packh= $packlist{$pn}; my $part=''; if (!$packh && $pn =~ m,^([^/]+)/(.*),) { $pn= $1; $part= $2; # just this part $packh= $packlist{$pn}; next unless ($packh); #? also need to cut pn from pl $packh->{userparts}= $part; # list? } if ($packh->{requires} && !$skiprequires) { my @xp= split(/[,\s]+/,$packh->{requires}); push(@pl,@xp); } } my %ph=(); my @pl2=(); foreach (@pl) { unless ($ph{$_}) { if(/install/) { unshift(@pl2, $_) } else { push(@pl2, $_); } } $ph{$_}++; } return @pl2; #dang, sorted list is fetch/update order - need to keep order in requires: # my %ph= map { $_,1; } @pl; # ? need package sort order? # @pl= sort _sortpack keys %ph; # return @pl; } sub _sortpack { return -1 if ($a =~ /install/); return 1 if ($b =~ /install/); return $a cmp $b; } sub getFileList { my ($fpat)= @_; if ($fpat =~ m,^(.*/)?([^/]+)$,) { my ($d,$fn)= ($1,$2); local(*D); if (opendir(D,$d)) { $fn =~ s,\*,.*,; my @fs= map { $d.$_; } grep(/^$fn$/,readdir(D)); closedir(D); return @fs; } } return (); } # sub editTemplateConfig { ## see run-apache .. # egrep '@[A-z]*@' "${APACHE_CONF}" >/dev/null # if [ $? -eq 0 ]; then # echo "Rewrite @ENV_VAR@ in ${APACHE_CONF}" # cat ${APACHE_CONF} | perl -pe 's/\@([^@]*)\@/$ENV{$1}/g;' > $confed # APACHE_CONF=$confed # echo "Using edited APACHE_CONF: $APACHE_CONF" # fi # } sub editConfigFiles { my ($ref)= @_; my %conflist= %$ref; my @confiles= sort keys %conflist; ## jun06 patch foreach my $ak (keys %config) { $ENV{$ak}= $config{$ak}; } foreach my $cf (@confiles) { my @fset; ## check also for $cf$sufDEFAULT ? if (-f "$ARGOS_ROOT/$cf") { @fset= ("$ARGOS_ROOT/$cf"); } elsif (-f "$ARGOS_ROOT/$cf$sufDEFAULT") { @fset= ("$ARGOS_ROOT/$cf"); } elsif ($cf =~ m,\*,) { @fset= getFileList("$ARGOS_ROOT/$cf"); unless(@fset) { @fset= getFileList("$ARGOS_ROOT/$cf$sufDEFAULT"); if (@fset) { foreach (@fset) { s/$sufDEFAULT$//; } } } } unless(@fset) { warn "# not found: config file $cf \n" if $verbose; next; } my @confs= split /[,\s]+/, $conflist{$cf}; my $ed= ''; foreach my $ck (@confs) { next unless $ck =~ /\S/; ## also add this @ARGOS_KEY@ edit method . mar04 ; need %config in %ENV # cat ${APACHE_CONF} | perl -pe 's/\@([^@]*)\@/$ENV{$1}/g;' > $confed my $old= $origconfig{$ck}; my $new= $config{$ck}; $ENV{$ck}= $config{$ck}; # jun06 ?? or see above unless (defined $old && defined $new) { warn "# config missing $ck = '$old' -> '$new' \n" if $verbose; } else { $old = '\\b'.$old if ($old =~ /^\w/); # check wordbounds $old = $old.'\\b' if ($old =~ /\w$/); $ed .= "s,$old,$new,g;"; } } unless ($ed) { warn "# no changes for config file $cf\n"; } else { $ed .= 's/\@([^@]*)\@/$ENV{$1}/g;'; # jun06 foreach my $cfn (@fset) { my $bak= $sufDEFAULT; if (-f "$cfn$sufDEFAULT") { ## need to use this for orig values doShell("/bin/cp", "-p", "$cfn$sufDEFAULT", "$cfn"); $bak= ".old"; } doShell("perl", "-pi$bak", "-e", "$ed", "$cfn"); ## need to add new file to package-folder/rsync.exclude.local ## and use these for update ## OR just start with $sufDEFAULT versions on source server ## and put list to exclude in packages.conf } } } } sub cleanval { local $_= shift; s/^\s*//; s/\s*$//; s,\\$,,; #trim ; drop \ continue line flag if (s/^(["'])//) { s/$1$//; } return $_; } sub readPackagesWithLocal { my ($confile, $rdefault, $nopacks)= @_; my %packs= readPackages($confile, $rdefault, $nopacks); if (!$nopacks && -f "$confile.local") { my %lpacks= readPackages("$confile.local", {}, $nopacks); foreach my $key (keys %lpacks) { next if ($key eq 'DEFAULT'); $packs{$key}= $lpacks{$key}; # replace all vals } } return %packs; } sub readPackages { my ($confile, $rdefault, $nopacks)= @_; my %packs; $configFileOk=0; local(*F); unless(open(F,"$confile")) { warn "config file missing: $confile\n" if $verbose; return %$rdefault; ## }; #? should we copy any $rdefault pack defs into new $pack{} ? my ($pack,$k,$v)= ('DEFAULT','',''); $packs{$pack}= {}; while() { next if(/^\s*[\#\!]/); # skip comments, etc. next unless(/^\s*\S/); # skip comments, etc. chomp; if (s/^\s+//) { # continue variable line $_= cleanval($_); $packs{$pack}->{$k} .= ' '.$_ if ($pack && $k); } elsif (/^([^\s=:]+)\s*[=:]\s*(.*)$/) { # must have value=(*.) to allow blanks ($k,$v)=($1,$2); $v= cleanval($v); if (!$nopacks && $k eq 'package') { $pack= $v; $packs{$pack}= {}; } $packs{$pack}->{$k}= $v; # preserve old install.conf.local package names if ($k eq 'package-alias') { $packs{$v}= $packs{$pack}; } } } close(F); $configFileOk=1; return %packs; } sub readConfig { my ($confile,$rdefault)= @_; my %pk= readPackages($confile,{},1); return (%pk) ? %{$pk{DEFAULT}} : %$rdefault; } sub availableSystems { my ($OS)= @_; my @oslist=(); # my $haveos=0; my $packh= $packlist{'argos-systems'}; my $from = $packh->{"url-$urlhost"} || $packh->{url} || "rsync://eugenes.org/argos/common/systems/"; my @syslist= `rsync $from`; foreach (@syslist) { chomp; my @cols = split ; next unless($cols[4] =~ /^\w/); push(@oslist, $cols[4]); } return @oslist; } sub makeLocalExclude { my ($OS,$fname,$roslist)= @_; # return if ($viewOnly); my @oslist= (ref $roslist) ? @$roslist : (); unless(@oslist) { @oslist= availableSystems(); } my $haveos = grep { $_ eq $OS } @oslist; #my $haveos = grep( /^$OS$/, @oslist); unless($viewOnly) { local(*T); open(T,">$fname") or die "writing $fname"; foreach (@oslist) { unless ( $_ eq $OS ) { print T $_ , "\/\n"; } } close(T); } return $haveos; } sub doRsync { my ($from, $to, $action, $excludes) = @_; my $ex; my @cmd = qw(rsync -a); #?? dont use -z to reduce server load ? if ($verbose) { if ($action =~ /^LIST/i) { print "# $action $from\n"; } else { warn "# $action $from\n"; } } if ($action =~ /^LIST_ALL/i) { # --Hardik print "# LOCAL_PATH:$to\n"; $to = ""; ##"/tmp"; # list-update to /tmp to get full file list, in update format } if($action =~ /UPDATE/i) { push @cmd, "-u","--modify-window=130","--include=*-distr"; ## FIXME rsync --include doesn't work reliably -- dgg push @cmd,"--delete" if ($cleanupd); #? need $update_excludes for saving local .conf edits!? } if (ref($excludes) && @$excludes) { push @cmd, @$excludes; } ## user choice for -v -b .. ## need to know if packages.conf is updated - keep always -v listing - or -M $package push @cmd, @rsyncopts if (@rsyncopts); unless($from) { warn "# bad rsync - missing source\n"; return -1; } push @cmd, $from; push @cmd, $to if ($to); # not for LIST return doShell(@cmd); } # Find System OS and return the name # aug05 - adding Sun x64-solaris10 to support # melon.% uname -a # SunOS melon 5.10 Generic i86pc i386 i86pc # uname -a intel-linux (redhat): # Linux lynx.bio.indiana.edu 2.4.21-27.0.2.ELsmp #1 SMP Wed Jan 12 23:35:44 EST 2005 i686 i686 i386 GNU/Linux # oat: SunOS oat 5.8 Generic_117350-18 sun4u sparc SUNW,Ultra-80 sub findOS { my $SYSOS=undef; my $TARGET_OS = "unknown"; my $OS = `uname`; chomp($OS); my $unamea= `uname -a`; my $unamer= `uname -r`; if ($OS eq 'SunOS') { $OS = ''; if ($unamea =~ /\bsparc/) { $OS='sparc-'; } elsif ($unamea =~ /\bi86pc/) { $OS='x64-'; } if ($unamer >= 5.10) { $OS .= 'solaris10'; } elsif ( $unamer !~ /^[1-4]/) { $OS .= 'solaris'; } } local $_= $OS; OSCASE: { if (/x64-solaris10/){ $TARGET_OS = "x64-solaris10"; last OSCASE; } #?? if (/sparc-solaris10/){ $TARGET_OS = "sparc-solaris10"; last OSCASE; } if (/sparc-solaris/){ $TARGET_OS = "sun-sparc-solaris"; last OSCASE; } if (/Linux/) { $TARGET_OS = "intel-linux"; last OSCASE; } if (/Darwin/) { $TARGET_OS = "macosx"; $SYSOS="apple-powerpc-darwin"; last OSCASE; } ## we are not likely adding these #if (/SunOS/) { $TARGET_OS = "sun4-solaris"; last OSCASE; } #if (/OSF1/) { $TARGET_OS = "osf"; last OSCASE; } #if (/ULTRIX/) { $TARGET_OS = "ultrix"; last OSCASE; } #if (/IRIX/) { $TARGET_OS = "sgi-irix"; last OSCASE; } #if (/AIX/) { $TARGET_OS = "ibm-aix"; last OSCASE; } #if (/HP-UX/) { $TARGET_OS = "hp-ux"; last OSCASE; } $TARGET_OS = $OS; print "Unsupported operating system: $OS"; } ## SYSOS - how system names library path; $TARGET_OS - srs name $SYSOS=$TARGET_OS unless($SYSOS); return (wantarray) ? ($SYSOS, $TARGET_OS) : $SYSOS; } #-------------------------------------------------------------- package ArgosInstallForm; use strict; use Getopt::Long; ## caller may lack CGI.pm on argos bootstrap #>>ERR>>use CGI qw(:standard *table *dl *TR *td *th); BEGIN{eval{require CGI; import CGI qw(:standard *table *dl *TR *td *th);};} use vars qw($title $HTMLHEADER $installer $vers); BEGIN { $title= "Argos Installation"; $vers= "Argos v".$ArgosInstaller::VERSION; # v0.5c $installer= 'installargos.pl'; ## FIXME $HTMLHEADER= 0; } =head1 ArgosInstallForm handle some of these for web form Commands: -info == list package info ** -install == install new server set ** create install.conf.local to taste -list == list package files at server ** some of this? verbose -update == update current server set -reconfig == change to new install.conf.local configuration Options: -root=/path/to/install == override ARGOS_ROOT=/bio/argos -package=a,b, ... == use specific packages ('all' for everyone) -rsopts=-v,-n ... == apply rsync flags (-n=show dry-run, -v=verbose) =cut sub downloadInstaller { my ($cg,$url)= @_; my $ARGOS_ROOT= $ArgosInstaller::ARGOS_ROOT; my $confile= "$ARGOS_ROOT/install/$installer"; print $cg->header( -XLayout => "", -type=>'application/octet-stream', -attachment => $installer, ); local(*F); unless(open(F,"$confile")) { print "config file missing: $confile
\n"; } else { print ; close(F); } } # make html page w/ URLs to this and to downloadInstaller? sub newConfig { my ($cg,$url)= @_; my $ARGOS_ROOT= $ArgosInstaller::ARGOS_ROOT; my $inroot= ArgosInstaller::installRoot(); my $inconf = "$inroot/install/$ArgosInstaller::NG_CONF"; my $confile= "$ARGOS_ROOT/install/$ArgosInstaller::NG_CONF_DEFAULT"; # print $cg->header(-type=>'text/plain'); ## need filename ! print $cg->header( ##-XLayout => "", -type=>'application/octet-stream', -attachment => $ArgosInstaller::NG_CONF, ); print "# Save As TEXT FILE= $inconf \n"; print "# Argos Package local configuration \n"; print "\n"; local(*F); unless(open(F,"$confile")) { print "config file missing: $confile
\n"; } else { my @confile= ; close(F); # print "
\n";    
    foreach (@confile) { 
      if (/^\s*(\w+)\s*[=:]\s*(.*)$/) {
        my ($k,$v)= ($1,$2);
        if (defined $cg->param($k)) {
          $v= $cg->param($k);
          }
        s/^\s*(\w+)\s*[=:]\s*(.*)$/$1 = $v/;
        }
      print;
      }
    # print "

\n"; } print "\n"; } sub editCrontab { my ($cg,$url)= @_; my $ARGOS_ROOT = $ArgosInstaller::ARGOS_ROOT; my $inroot= ArgosInstaller::installRoot(); #? $confroot == config{ARGOS_ROOT} my $cronfile= "$ARGOS_ROOT/install/$ArgosInstaller::NG_CRONTAB"; local(*F); unless(open(F,$cronfile)) { htmlHeader($cg); print "crontab file missing: $cronfile
\n"; return; } print $cg->header( -XLayout => "", -type=>'application/octet-stream', -attachment => $ArgosInstaller::NG_CRONTAB, ); my @adds= $cg->param("addpack"); my @cronfile= ; close(F); foreach (@cronfile) { while (/\@(\w+)\@/) { my ($k)= ($1); if ($k eq 'ARGOS_SERVICE') { my $t= $_; foreach my $v (grep(!/argos-/,@adds)) { $_= $t; s/\@$k\@/$v/; print; } $_=''; } else { my $v= $ArgosInstaller::config{$k} || 'UNDEFINED'; if ($k eq 'ARGOS_ROOT') { $v= $inroot; } if (defined $cg->param($k)) { $v= $cg->param($k); } ##elsif (defined $ENV{$k}) { $v= $ENV{$k}; } #?? s/\@$k\@/$v/; } } print; } print "\n"; } sub editConfig { my ($cg,$url)= @_; my $ARGOS_ROOT = $ArgosInstaller::ARGOS_ROOT; my $inroot= ArgosInstaller::installRoot(); my $croot= $ARGOS_ROOT; my $confile= "$ARGOS_ROOT/install/$ArgosInstaller::NG_CONF_DEFAULT"; my $inconf= "$inroot/install/$ArgosInstaller::NG_CONF"; htmlHeader($cg); print "

Package configuration

\n"; # print "FILE=$confile

\n"; my @addpack= $cg->param("addpack"); # printConfig($confile); # if (param('Configure')); my $table = TR( {-bgcolor=>'#cccccc'}, th( "Key"), th( "value"), )."\n"; my @optpack=(); foreach my $pack (@ArgosInstaller::packlist) { my $packh= $ArgosInstaller::packlist{$pack}; next if ($pack ne $packh->{'package'} && $packh->{'package-alias'}); my $isrequired= ($packh->{'is-required'} =~ /true|1/i); push(@optpack,$pack) unless($isrequired); unshift(@addpack,$pack) if ($isrequired && grep(/^$pack$/,@addpack) == 0); } local(*F); unless(open(F,"$confile")) { print "config file missing: $confile
\n"; } else { my @confile= ; close(F); # print "
\n"; print @confile;  print "

\n"; my $notes; foreach (@confile){ if (/^#n\w*(.*)$/) { $notes .= $1."
\n"; } next unless(/^\s*(\w+)\s*[=:]\s*(.*)$/); if ($notes) { #$table .= TR( td({-colspan=>2},"NOTE: ".$notes))."\n"; $table .= TR({-bgcolor=>'#f0f0f0'}, td({-align=>'right'},'Note'), td($notes))."\n"; $notes= ''; } my ($k,$v)= ($1,$2); if ($k =~ /^ARGOS_ROOT$/) { $croot= $v; $v= $inroot; } elsif ($k =~ /^PACKAGES$/) { $v= '"'.join(" ",@addpack).'"'; } elsif ($k =~ /^TOMCAT_SERVERS$/) { ## added feb04 $v= '"'.join(" ", grep(!/argos-/, @addpack) ).'"'; } elsif ($k =~ /^(\w+)_CONF$/) { my $pk= lc($1); if (grep(/^$pk$/,@addpack)) { $v= '';} else { $v= '#'.$k; } } elsif ($v =~ /$croot/) { $v =~ s/$croot/$inroot/g; } #need for PERLLIB=/root/a:/root/b:/root/c #elsif ($v =~ /^$croot/) { $v =~ s/$croot/$inroot/; } #?? ## filter out unwanted package_xxx configs ? if ($k =~ /^(\w+)_/) { my $pk= lc($1); my $skipit= ( grep(/^$pk$/,@optpack) != 0 && grep(/^$pk$/,@addpack) == 0) ; next if $skipit; } $v= $cg->textfield( -size=>50, -name=>$k,-value=>$v); $table .= TR( td($k), td($v))."\n"; } # close(F); print $cg->start_form(-name=>'argosconfig', -action=>$url),"\n"; print "Installation FILE=$inconf", "  ",$cg->submit(-name=>'Create'),"

\n"; print table( {-border=>1, -cellpadding=>4, -bgcolor=>'white', width=>'100%' }, $table); # print $cg->hidden(-name=>'option',-value=>'edit'); print $cg->hidden(-name=>'inroot',-value=>$inroot); print $cg->hidden(-name=>'config',-value=>$confile); print $cg->end_form(),"\n"; } # print $cg->end_html(),"\n"; print "


\n"; } sub printInfo { my ($cg, $url,@packs)= @_; @packs= @ArgosInstaller::packlist unless(@packs); htmlHeader($cg); print "

Package info

\n"; # print "
\n";
  foreach my $pack (@packs) {
    # ArgosInstaller::packageInfo($pack);

    my $table =  TR( {-bgcolor=>'#cccccc'}, th( "Key"), th( "value"), )."\n";
    my $packh= $ArgosInstaller::packlist{$pack};
    # print "package: $pack\n";
    $table .= TR( td('Package'), td($pack));
    foreach my $k (sort keys %$packh) {
      next if ($k eq 'package');
      my $v= $packh->{$k};
      $v= reqLink($url, $v) if ($k =~ /^(requires|optional)$/);
      $v= listLink($url, $pack, $v, $cg) if ($k =~ /^(url)$/);
      $table .= TR( td($k), td($v))."\n";
     }
    print table( {-border=>1, -cellpadding=>4, -bgcolor=>'white', width=>'100%' }, $table);
    print "

\n"; } # print "

\n"; } sub printList { my ($cg, $url,@packs)= @_; @packs= @ArgosInstaller::packlist unless(@packs); htmlHeader($cg); $ArgosInstaller::viewOnly= 0; #?? FIXME - very slow printing compared to cmdline ## is sending info to stderr ! $|=1; # unbuffer print foreach my $pack (@packs) { my $packh= $ArgosInstaller::packlist{$pack}; print "

Package Listing: $pack

\n"; # print "LIST ".$packh->{url}."
\n"; print "
\n";
    ## not working well yet??
    my $listupdate= $cg->param('update');
    if ( $cg->param('debug') ) { 
      ArgosInstaller::listPackageForCgi($pack,$listupdate);  ## Need to trap long lists and page them??
      # ArgosInstaller::listPackage($pack,'LIST_UPDATE');  ## Need to trap long lists and page them??
      }
    else { 
        print "Not working properly yet... 
"; print "try command-line 'rsync -nav --stats $packh->{url}' to list
\n"; } print "

\n"; } } sub reqLink { my ($url,$req)= @_; # $packh->{'requires'} my $rh=''; my @req= split(" ",$req); # $packh->{'requires'}); foreach my $r (@req) { $rh .= a({-href=>"$url?option=info&package=$r"},$r)." "; } return $rh; } sub listLink { my ($url,$pack,$req, $cg)= @_; # $packh->{'requires'} my $rh=''; my @req= split(" ",$req); # $packh->{'requires'}); foreach my $r (@req) { # $rh .= a({-href=>"$url?option=list&package=$pack"},$r)." "; ## change this to cgi form button so robots dont traverse $rh .= $cg->start_form(-method=>'GET',-name=>'argosinstall', -action=>$url); $rh .= $cg->hidden(-name=>'package',-value=>$pack); $rh .= $cg->submit(-name=>'option',-value=>'list'); $rh .= $r; $rh .= $cg->end_form()."\n"; } return $rh; } sub htmlHeader { my $cg = shift; print $cg->header,"\n", $cg->start_html(-title => $title, ),"\n", $cg->h2($title),"\n" unless($HTMLHEADER++); } sub requirements { my ($cg, $url, $inroot)= @_; print <<"EOF";

Requirements

Unix systems supported are: Sun Solaris v8 & v9; Intel Linux (redhat v8, 9 tested); Apple MacOSX v10.2+
These packages are required, and commonly are preinstalled except for Solaris EOF } sub instructions { my ($cg, $url, $inroot)= @_; print "

Installation steps

\n"; my $aport= $ArgosInstaller::config{APACHE_PORT}; print "
    \n
  1. Download: ", $cg->a({-href=>"$url?option=getinstaller"},$installer)."
    \n", " Also please read Readme-current.txt\n", " and argos-system-management.txt\n", "

    \n"; print "

  2. Bootstrap installation: ", " From command line, Run (change $inroot as desired)
    ", " 'perl installargos.pl -root=$inroot -install -v' ", "

    \n"; print "

  3. Configure: ", " Check desired packages below, edit Installation path, and press Configure button", "
    Installation path: ", $cg->textfield( -size=>50, -name=>'inroot',-value=>$inroot), # user opt! "    ",$cg->submit(-name=>'Configure'), "

    \n"; print "

  4. Install full data, software set: ", " Run
    'cd $inroot; install/install -install -v'
    \n", " This can take a while, use 'install/install -install -v >& log.install &' for background
    \n", "

    \n"; print "

  5. Check installation, Start web server, Check logs/: ", " Run
    'install/run-apache start'
    \n", " The Argos web server should then be available at http://localhost:$aport/ \n", " with installed packages at configured ports (status in logs/)
    \n", "

    \n"; print "

  6. Update regularly: ", " Run
    'install/install -update [-v -n] [-rsopts=-v,-n]'
    \n", " See this example ",$cg->submit(-name=>'Crontab')," entry for server management.\n", ""; # " These '-v' flags mean verbose output, ", # " and '-n' means no changes, that is show but don't do updates,", # " -rsopts=-v,-n will list all files to be updated without doing the update.", print "
\n"; } sub installForm { my $cg = new CGI; # print $cg->header,"\n", # $cg->start_html(-title => $title, ),"\n", # $cg->h2($title),"\n"; ## ! use findOS for browser info; set JAVA_HOME, other os-specifics for config? ## aug07: block rsync, other system calls when this is run at IUBio server my $isiubio=($ENV{SERVER_NAME}=~/iubio/ or $ENV{SERVER_ADDR}=~/129.79.225./) ? 1 : 0; my @savearg= @ARGV; unshift( @ARGV, -root => ($cg->param('webroot') || '..')); #?? installroot ? if ($cg->param('inroot')) { unshift( @ARGV, -inroot => $cg->param('inroot')); #?? installroot ? } ## -package => ($cg->param('package') || 'argos-root'), ## ($cg->param('option') || '-info'), my $ok= GetOptions(%ArgosInstaller::runopts); ArgosInstaller::getConfigs($ok) unless($isiubio); my $webroot= $ArgosInstaller::ARGOS_ROOT; my $inroot= ArgosInstaller::installRoot(); my $confroot= $ArgosInstaller::config{ARGOS_ROOT}; #?? which to use? my $url= $cg->url(-relative=>1,-path_info=>1); if (param('option') eq 'info') { printInfo( $cg, $url, param('package') ); } elsif (param('option') eq 'list') { printList( $cg, $url, param('package') ); } elsif (param('option') eq 'getinstaller') { downloadInstaller( $cg, $url); } elsif (param('Configure')) { editConfig($cg, $url); } elsif (param('Crontab')) { editCrontab($cg, $url, $confroot); } elsif (param('Create')) { newConfig( $cg, $url); } else { ## change this - make 1st page = instructions, with steps ## 1. htmlHeader($cg); requirements($cg); print $cg->start_form(-name=>'argosinstall', -action=>$url),"\n"; instructions($cg, $url, $inroot); ##$confroot # print "Installation path: ", # $cg->textfield( -size=>50, -name=>'inroot',-value=>$inroot), # user opt! # "    ",$cg->submit(-name=>'Configure'),"

\n"; print "Main packages to install: ", "\n"; my $table = TR( {-bgcolor=>'#cccccc'}, th( "Package"), th( "Local path"), th( "Disk use"), th( "Description"), th( { -width=>"25%"}, "Requires"), ); # foreach my $pack (@ArgosInstaller::packlist) { my $packh= $ArgosInstaller::packlist{$pack}; next unless ($packh->{'is-main'} =~ /true|1/i); next if ($pack ne $packh->{'package'} && $packh->{'package-alias'}); my $isrequired= ($packh->{'is-required'} =~ /true|1/i); my $rh= reqLink($url, $pack .' '.$packh->{'requires'}); $table .= "\n" . TR( td( checkbox( -label => $pack, -name => 'addpack', -value=>$pack, -override => 1, -checked => param($pack) || $isrequired , ($isrequired ? '-disabled' : ''), # this is bad - form doesnt send value? )), td( $packh->{'localpath'} ), td( $packh->{'size'} || $packh->{'est-size'} || '--' ), td( $packh->{'description'} ), td( { -width=>"30%"}, $rh), #$packh->{'requires'} ); } print table( {-border=>1, -bgcolor=>'white', width=>'100%' }, $table); print $cg->end_form(),"\n"; } print $cg->end_html(),"\n" if ($HTMLHEADER); } 1; __END__