From bd9e6617827818fd043452c08c606f07b78014a0 Mon Sep 17 00:00:00 2001 From: toma Date: Wed, 25 Nov 2009 17:56:58 +0000 Subject: Copy the KDE 3.5 branch to branches/trinity for new KDE 3.5 features. BUG:215923 git-svn-id: svn://anonsvn.kde.org/home/kde/branches/trinity/kdesdk@1054174 283d02a7-25f6-0310-bc7c-ecb5cbfe19da --- scripts/kdesvn-build | 4286 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 4286 insertions(+) create mode 100755 scripts/kdesvn-build (limited to 'scripts/kdesvn-build') diff --git a/scripts/kdesvn-build b/scripts/kdesvn-build new file mode 100755 index 00000000..fcd3970b --- /dev/null +++ b/scripts/kdesvn-build @@ -0,0 +1,4286 @@ +#!/usr/bin/perl -w + +#Pod documentation: + +=head1 NAME + +=over + +=item B - automate the kde svn build process + +=back + +=head1 SYNOPSIS + +=over + +=item B I<[options]...> I<[modules]...> + +=back + +=head1 DESCRIPTION + +The B script is used to automate the download, build, +and install process for KDE (using Subversion). + +It is recommended that you first setup a F<.kdesvn-buildrc> file +in your home directory. Please refer to B help file +in KDE help for information on how to write F<.kdesvn-buildrc>, +or consult the sample file which should have been included +with this program. If you don't setup a F<.kdesvn-buildrc>, a +default set of options will be used, and a few modules will be +built by default. + +After setting up F<.kdesvn-buildrc>, you can run this program from +either the command-line or from cron. It will automatically +download the modules from Subversion, create the build +system, and configure and make the modules you tell it to. +You can use this program to install KDE as well, +if you are building KDE for a single user. Note that B +will try to install the modules by default. + +If you DO specify a package name, then your settings will still be +read, but the script will try to build / install the package +regardless of F<.kdesvn-buildrc> + +kdesvn-build reads options in the following order: + +=over + +=item 1. From the command line. + +=item 2. From the file F in the current directory. Note that + the file is not a hidden file. + +=item 3. From the file F<~/.kdesvn-buildrc>. + +=item 4. From a set of internal options. + +=back + +This utility is part of the KDE Software Development Kit. + +=head1 OPTIONS + +=over + +=item B<--quiet>, B<-q> + +With this switch kdesvn-build will only output a general overview of the build +process. Progress output is still displayed if available. + +=item B<--really-quiet> + +With this switch only warnings and errors will be output. + +=item B<--verbose>, B<-v> + +Be very detailed in what is going on, and what actions kdesvn-build is taking. +Only B<--debug> is more detailed. + +=item B<--no-svn> + +Skip contacting the Subversion server. + +=item B<--no-build> + +Skip the build process. + +=item B<--no-install> + +Don't automatically install after build. + +=item B<--svn-only> + +Update from Subversion only (Identical to B<--no-build> at this point). + +=item B<--build-only> + +Build only, do not perform updates or install. + +=item B<--rc-file=EfilenameE> + +Read configuration from filename instead of default. + +=item B<--debug> + +Activates debug mode. + +=item B<--pretend>, B<-p> + +Do not contact the Subversion server, run make, or create / delete files +and directories. Instead, output what the script would have done. + +=item B<--nice=EvalueE> + +Allow you to run the script with a lower priority. The default value is +10 (lower priority by 10 steps). + +=item B<--prefix=/kde/path> + +This option is a shortcut to change the setting for kdedir from the +command line. It implies B<--reconfigure>. + +=item B<--color> + +Add color to the output. + +=item B<--no-color> + +Remove color from the output. + +=item B<--resume> + +Tries to resume the make process from the last time the script was run, +without performing the Subversion update. + +=item B<--resume-from=EpkgE> + +Starts building from the given package, without performing the Subversion +update. + +=item B<--revision=ErevE>, B<-r=ErevE> + +Forces update to revision from Subversion. + +=item B<--refresh-build> + +Start the build from scratch. This means that the build directory for the +module B before make -f Makefile.cvs is run again. You can +use B<--recreate-configure> to do the same thing without deleting the module +build directory. + +=item B<--reconfigure> + +Run configure again, but don't clean the build directory or re-run +make -f Makefile.cvs. + +=item B<--recreate-configure> + +Run make -f Makefile.cvs again to redo the configure script. The build +directory is not deleted. + +=item B<--no-rebuild-on-fail> + +Do not try to rebuild a module from scratch if it failed building. Normally +kdesvn-build will try progressively harder to build the module before giving +up. + +=item B<--build-system-only> + +Create the build infrastructure, but don't actually perform the build. + +=item B<--install> + +Try to install the packages passed on the command line, or all packages in +F<~/.kdesvn-buildrc> that don't have manual-build set. Building and +Subversion updates are not performed. + +=item B<--EoptionE=> + +Any unrecognized options are added to the global configuration, overriding +any value that may exist. + +For example, B<--svn-server=http://path.to.svn.server/> would change the +setting of the global B option for this instance of kdesvn-build. + +=item B<--EmoduleE,EoptionE=> + +Likewise, allow you to override any module specific option from the +command line. + +Example: B<--kdelibs,use-unsermake=false> would disable unsermake for the +kdelibs module. + +=item B<--help> + +Display the help and exit. + +=item B<--author> + +Output the author(s)'s name. + +=item B<--version> + +Output the program version. + +=back + +=head1 EXAMPLES + +=over + +=item B + +=item B I<--no-svn kdelibs> + +=item B I<--refresh-build> I + +=back + +=head1 BUGS + +Since kdesvn-build doesn't generally save information related to the build and +prior settings, you may need to manually re-run kdesvn-build with a flag like +B<--recreate-configure> if you change some options, including B. + +Please use KDE bugzilla at http://bugs.kde.org for information and +reporting bugs. + +=head1 SEE ALSO + +You can find additional information at B home page, +F, or using kdesvn-build +docbook documentation, using the help kioslave, F. + +=head1 AUTHOR + +Michael Pyne + +Man page written by: +Carlos Leonhard Woelz + +=cut + +# Script to handle building KDE from Subversion. All of the configuration is +# stored in the file ~/.kdesvn-buildrc. +# +# Please also see the documentation that should be included with this program, +# in doc.html +# +# Copyright (c) 2003, 2004, 2005 Michael Pyne. +# Home page: http://kdesvn-build.kde.org/ +# +# You may use, alter, and redistribute this software under the terms +# of the GNU General Public License, v2 (or any later version). +# +# TODO: It would be better to have lockfiles in each directory as it's +# being updated, instead of having one big lock for the script. + +use strict; +use warnings; +use Fcntl; # For sysopen constants +use POSIX 'strftime'; +use File::Find; # For our lndir reimplementation. +use Errno qw(:POSIX); + +# Debugging level constants. +use constant { + DEBUG => 0, + WHISPER => 1, + INFO => 2, + NOTE => 3, + WARNING => 4, + ERROR => 5, +}; + +# Some global variables +# Remember kids, global variables are evil! I only get to do this +# because I'm an adult and you're not! :-P +# Options that start with a # will replace values with the same name, +# if the option is actually set. +my %package_opts = ( + 'global' => { + "apidox" => "", + "apply-qt-patches" => "", + "binpath" => "/bin:/usr/bin:/usr/X11R6/bin:/usr/local/bin", + "branch" => "", + "build-dir" => "build", + "build-system-only" => "", + "checkout-only" => "", + "configure-flags" => "--enable-debug", + "colorful-output" => 1, # Use color by default. + "cxxflags" => "-pipe", + "debug" => "", + "debug-level" => INFO, + "dest-dir" => '${MODULE}', # single quotes used on purpose! + "disable-agent-check" => 0, # If true we don't check on ssh-agent + "do-not-compile" => "", + "email-address" => "", + "email-on-compile-error" => "", + "install-after-build" => "1", # Default to true + "inst-apps" => "", + "kdedir" => "$ENV{HOME}/kde", + "libpath" => "", + "log-dir" => "log", + "make-install-prefix" => "", # Some people need sudo + "make-options" => "-j2", + "manual-build" => "", + "manual-update" => "", + "module-base-path" => "", # Used for tags and branches + "niceness" => "10", + "no-svn" => "", + "no-rebuild-on-fail" => "", + "override-url" => "", + "prefix" => "", # Override installation prefix. + "pretend" => "", + "qtdir" => "$ENV{HOME}/kdesvn/build/qt-copy", + "reconfigure" => "", + "recreate-configure" => "", + "refresh-build" => "", + "remove-after-install"=> "none", # { none, builddir, all } + "revision" => 0, + "set-env" => { }, # Hash of environment vars to set + "source-dir" => "$ENV{HOME}/kdesvn", + "stop-on-failure" => "", + "svn-server" => "svn://anonsvn.kde.org/home/kde", + "tag" => "", + "unsermake-options" => "--compile-jobs=2 -p", + "unsermake-path" => "unsermake", + "use-unsermake" => "1", # Default to true now, we may need a blacklist + } +); + +# This is a hash since Perl doesn't have a "in" keyword. +my %ignore_list; # List of packages to refuse to include in the build list. + +# update and build are lists since they support an ordering, which can't be +# guaranteed using a hash unless I want a custom sort function (which isn't +# necessarily a horrible way to go, I just chose to do it this way. +my @update_list; # List of modules to update/checkout. +my @build_list; # List of modules to build. + +# Dictionary of lists of failed modules, keyed by the name of the operation +# that caused the failure (e.g. build). Note that output_failed_module_lists +# uses the key name to display text to the user so it should describe the +# actual category of failure. You should also add the key name to +# output_failed_module_lists since it uses its own sorted list. +my @fail_display_order = qw/build update install/; +my %fail_lists = ( + 'build' => [ ], + 'install' => [ ], + 'update' => [ ], +); + +my $install_flag; # True if we're in install mode. +my $BUILD_ID; # Used by logging subsystem to create a unique log dir. +my $LOG_DATE; # Used by logging subsystem to create logs in same dir. +my @rcfiles = ("./kdesvn-buildrc", "$ENV{HOME}/.kdesvn-buildrc"); +my $rcfile; # the file that was used; set by read_options + +# Colors +my ($RED, $GREEN, $YELLOW, $NORMAL, $BOLD) = ("") x 5; + +# Subroutine definitions + +# I swear Perl must be the only language where the docs tell you to use a +# constant that you'll never find exported without some module from CPAN. +use constant PRIO_PROCESS => 0; + +# I'm lazy and would rather write in shorthand for the colors. This sub +# allows me to do so. Put it right up top to stifle Perl warnings. +sub clr($) +{ + my $str = shift; + + $str =~ s/g\[/$GREEN/g; + $str =~ s/]/$NORMAL/g; + $str =~ s/y\[/$YELLOW/g; + $str =~ s/r\[/$RED/g; + $str =~ s/b\[/$BOLD/g; + + return $str; +} + +# Subroutine which returns true if pretend mode is on. Uses the prototype +# feature so you don't need the parentheses to use it. +sub pretending() +{ + return get_option('global', 'pretend'); +} + +# Subroutine which returns true if debug mode is on. Uses the prototype +# feature so you don't need the parentheses to use it. +sub debugging() +{ + return get_option('global', 'debug-level') <= DEBUG; +} + +# The next few subroutines are used to print output at different importance +# levels to allow for e.g. quiet switches, or verbose switches. The levels are, +# from least to most important: +# debug, whisper, info (default), note (quiet), warning (very-quiet), and error. +# +# You can also use the pretend output subroutine, which is emitted if, and only +# if pretend mode is enabled. +# +# clr is automatically run on the input for all of those functions. +# Also, the terminal color is automatically reset to normal as well so you don't +# need to manually add the ] to reset. + +# Subroutine used to actually display the data, calls clr on each entry first. +sub print_clr(@) +{ + print clr $_ foreach (@_); + print clr "]\n"; +} + +sub debug(@) +{ + print_clr @_ if debugging; +} + +sub whisper(@) +{ + print_clr @_ if get_option('global', 'debug-level') <= WHISPER; +} + +sub info(@) +{ + print_clr @_ if get_option('global', 'debug-level') <= INFO; +} + +sub note(@) +{ + print_clr @_ if get_option('global', 'debug-level') <= NOTE; +} + +sub warning(@) +{ + print_clr @_ if get_option('global', 'debug-level') <= WARNING; +} + +# This sub has the additional side effect of printing the errno value if it +# is set. +sub error(@) +{ + print STDERR (clr $_) foreach (@_); + print " $!\n" if $!; +} + +sub pretend(@) +{ + print_clr @_ if pretending; +} + +# Subroutine to handle removing the lock file upon receiving a signal +sub quit_handler +{ + note "Signal received, terminating."; + finish(5); +} + +# Subroutine that returns the path of a file used to output the results of the +# build process. It accepts one parameter, which changes the kind of file +# returned. If the parameter is set to 'existing', then the file returned is +# the latest file that exists, or undef if no log has been created yet. This +# is useful for the --resume mode. All other values will return the name if a +# file that does not yet exist. +# +# All files will be stored in the log directory. +sub get_output_file +{ + my $logdir; + my $mode; + $mode = shift or $mode = ''; + my $fname; + + debug "get_output_file in mode $mode"; + + if ($mode eq 'existing') + { + # There's two ways of finding the old file. Searching backwards with + # valid combinations of the date and build id, or just reading in the + # name from a known file or location. Since the latter option is much + # easier, that's what I'm going with. Note that this depends on the + # latest symlink being in place. + $logdir = get_subdir_path ('global', 'log-dir'); + $fname = "$logdir/latest/build-status"; + + debug "Old build status file is $fname"; + + # The _ at the end returns the cached file stats to avoid multiple + # stat() calls. + return "" if not -e $fname or not -r _; + + return $fname; + } + + # This call must follow the test above, because it changes the 'latest' + # symlink leading to failures later. + $logdir = get_log_dir('global'); + + $fname = "$logdir/build-status"; + debug "Build status file is $fname"; + + return $fname; +} + +# Subroutine to retrieve a subdirecty path for the given module. +# First parameter is the name of the module, and the second +# parameter is the option key (e.g. build-dir or log-dir). +sub get_subdir_path +{ + my $module = shift; + my $option = shift; + my $dir = get_option($module, $option); + + # If build-dir starts with a slash, it is an absolute path. + return $dir if $dir =~ /^\//; + + # If it starts with a tilde, expand it out. + if ($dir =~ /^~/) + { + $dir =~ s/^~/$ENV{'HOME'}/; + } + else + { + # Relative directory, tack it on to the end of $kdesvn. + my $kdesvndir = get_kdesvn_dir(); + $dir = "$kdesvndir/$dir"; + } + + return $dir; +} + +# Subroutine to return the name of the destination directory for the checkout +# and build routines. Based on the dest-dir option. The return value will be +# relative to the src/build dir. The user may use the '$MODULE' or '${MODULE}' +# sequences, which will be replaced by the name of the module in question. +# +# The first parameter should be the module name. +sub get_dest_dir +{ + my $module = shift; + my $dest_dir = get_option($module, 'dest-dir'); + + $dest_dir =~ s/(\${MODULE})|(\$MODULE\b)/$module/g; + + return $dest_dir; +} + +# Convienience subroutine to get the source root dir. +sub get_kdesvn_dir +{ + return get_option ('global', 'source-dir'); +} + +# Function to work around a Perl language limitation. +# First parameter is the list to search. +# Second parameter is the value to search for. +# Returns true if the value is in the list +sub list_has(\@$) +{ + my ($list_ref, $value) = @_; + return scalar grep ($_ eq $value, @{$list_ref}); +} + +# Subroutine to return the branch prefix. i.e. the part before the branch name +# and module name. +# +# The first parameter is the module in question. +# The second parameter should be 'branches' if we're dealing with a branch or +# 'tags' if we're dealing with a tag. +# +# Ex: 'kdelibs' => 'branches/KDE' +# 'kdevelop' => 'branches/kdevelop' +sub branch_prefix +{ + my $module = shift; + my $type = shift; + + # These modules seem to have their own subdir in /tags. + my @tag_components = qw/arts koffice amarok kst qt taglib/; + + # The map call adds the kde prefix to the module names because I don't feel + # like typing them all in. kdevelop and konstruct are special cases. + my @kde_module_list = ((map {'kde' . $_} qw/-i18n -common accessibility + addons admin artwork base bindings edu games graphics libs + multimedia network nonbeta pim sdk toys utils webdev/), 'kdevelop', + 'konstruct'); + + # KDE proper modules seem to use this pattern. + return "$type/KDE" if list_has(@kde_module_list, $module); + + # If we doing a tag just return 'tags' because the next part is the actual + # tag name, which is added by the caller, unless the module has its own + # subdirectory in /tags. + return "$type" if $type eq 'tags' and not list_has(@tag_components, $module); + + # Everything else. + return "$type/$module"; +} + +# Subroutine to return a module URL for a module using the 'branch' option. +# First parameter is the module in question. +# Second parameter is the type ('tags' or 'branches') +sub handle_branch_tag_option +{ + my ($module, $type) = @_; + my $svn_server = get_option($module, 'svn-server'); + my $branch = branch_prefix($module, $type); + my $branchname = get_option($module, 'tag'); + + if($type eq 'branches') + { + $branchname = get_option($module, 'branch'); + } + + # Remove trailing slashes. + $svn_server =~ s/\/*$//; + + return "$svn_server/$branch/$branchname/$module"; +} + +# Subroutine to return the appropriate SVN URL for a given module, based on +# the user settings. For example, 'kdelibs' -> https://svn.kde.org/home/kde/trunk/KDE/kdelibs +sub svn_module_url +{ + my $module = shift; + my $svn_server = get_option($module, 'svn-server'); + my $branch = get_option($module, 'module-base-path'); + + # Allow user to override normal processing of the module in a few ways, + # to make it easier to still be able to use kdesvn-build even when I + # can't be there to manually update every little special case. + if(get_option($module, 'override-url')) + { + return get_option($module, 'override-url'); + } + + if(get_option($module, 'tag')) + { + return handle_branch_tag_option($module, 'tags'); + } + + if(get_option($module, 'branch')) + { + return handle_branch_tag_option($module, 'branches'); + } + + # The following modules are in /trunk, not /trunk/KDE. There are others, + # but there are the important ones. The hash is associated with the value + # 1 so that we can do a boolean test by looking up the module name. + my @non_trunk_modules = qw(extragear kdenonbeta kdesupport koffice + playground qt-copy valgrind KDE kdereview www l10n); + + my $module_root = $module; + $module_root =~ s/\/.*//; # Remove everything after the first slash + + if (not $branch) + { + $branch = 'trunk/KDE'; + $branch = 'trunk' if list_has(@non_trunk_modules, $module_root); + } + + $branch =~ s/^\/*//; # Eliminate / at beginning of string. + $branch =~ s/\/*$//; # Likewise at the end. + + # Remove trailing slashes. + $svn_server =~ s/\/*$//; + + return "$svn_server/$branch/$module"; +} + +# Convienience subroutine to return the build directory for a module. Use +# this instead of get_subdir_path because this special-cases modules for you, +# such as qt-copy. +# TODO: From what I hear this hack is no longer necessary. Investigate this. +sub get_build_dir +{ + my $module = shift; + + # It is the responsibility of the caller to append $module! + return get_kdesvn_dir() if ($module eq 'qt-copy') and not get_option('qt-copy', 'use-qt-builddir-hack'); + return get_subdir_path($module, 'build-dir'); +} + +# Subroutine to return a list of the different log directories that are used +# by the different modules in the script. +sub get_all_log_directories +{ + my @module_list = keys %package_opts; + my %log_dict; + + # A hash is used to track directories to avoid duplicate entries. + unshift @module_list, "global"; + $log_dict{get_subdir_path($_, 'log-dir')} = 1 foreach @module_list; + + debug "Log directories are ", join (", ", keys %log_dict); + return keys %log_dict; +} + +# Subroutine to determine the build id for this invocation of the script. The +# idea of a build id is that we want to be able to run the script more than +# once in a day and still retain each set of logs. So if we run the script +# more than once in a day, we need to increment the build id so we have a +# unique value. This subroutine sets the global variable $BUILD_ID and +# $LOG_DATE for use by the logging subroutines. +sub setup_logging_subsystem +{ + my $min_build_id = "00"; + my $date = strftime "%F", localtime; # ISO 8601 date + my @log_dirs = get_all_log_directories(); + + for (@log_dirs) + { + my $id = "01"; + $id++ while -e "$_/$date-$id"; + + # We need to use a string comparison operator to keep + # the magic in the ++ operator. + $min_build_id = $id if $id gt $min_build_id; + } + + $LOG_DATE = $date; + $BUILD_ID = $min_build_id; +} + +# Convienience subroutine to return the log directory for a module. +# It also creates the directory and manages the 'latest' symlink. +# +# Returns undef on an error, or the name of the directory otherwise. +sub get_log_dir +{ + my $module = shift; + my $logbase = get_subdir_path($module, 'log-dir'); + my $logpath = "$logbase/$LOG_DATE-$BUILD_ID/$module"; + + $logpath = "$logbase/$LOG_DATE-$BUILD_ID" if $module eq 'global'; + + debug "Log directory for $module is $logpath"; + + if (not -e $logpath and not pretending and not super_mkdir($logpath)) + { + error "Unable to create log directory r[$logpath]"; + return undef; + } + + # Add symlink to the directory. + # TODO: This probably can result in a few dozen unnecessary calls to + # unlink and symlink, fix this. + if (not pretending) + { + unlink("$logbase/latest") if -l "$logbase/latest"; + symlink("$logbase/$LOG_DATE-$BUILD_ID", "$logbase/latest"); + } + + return $logpath; +} + +# This function returns true if the given option doesn't make sense with the +# given module. +# blacklisted($module, $option) +sub blacklisted +{ + my ($module, $option) = @_; + + # Known to not work. + my @unsermake_ban_list = qw/valgrind kde-common qt-copy kdebindings/; + + return list_has(@unsermake_ban_list, $module) if ($option eq 'use-unsermake'); + return 0; +} + +# This subroutine returns an option value for a given module. Some +# globals can't be overridden by a module's choice. If so, the +# module's choice will be ignored, and a warning will be issued. +# +# Option names are case-sensitive! +# +# First parameter: Name of module +# Second paramenter: Name of option +sub get_option +{ + my $module = shift; + my $option = shift; + my $global_opts = $package_opts{'global'}; + my $defaultQtCopyArgs = '-qt-gif -plugin-imgfmt-mng -thread -no-exceptions -debug -dlopen-opengl -plugin-sql-sqlite'; + my @lockedOpts = qw(source-dir svn-server qtdir libpath binpath kdedir + pretend disable-agent-check); + + # These options can't override globals + if (list_has(@lockedOpts, $option) or $module eq 'global') + { + return ${$global_opts}{"#$option"} if exists ${$global_opts}{"#$option"}; + return ${$global_opts}{$option}; + } + + # Don't even try this + return 0 if blacklisted($module, $option); + + my $ref = $package_opts{$module}; + + # Check for a sticky option + return $$ref{"#$option"} if exists $$ref{"#$option"}; + + # Next in order of precedence + if (defined ${$global_opts}{"#$option"} and not + ($module eq 'qt-copy' and $option eq 'configure-flags')) + { + return ${$global_opts}{"#$option"}; + } + + # No sticky options left. + # Configure flags and CXXFLAGS are appended to the global option + if (($module ne 'qt-copy' && $option eq 'configure-flags') + || $option eq 'cxxflags') + { + my $value = ${$global_opts}{$option}; + + if(defined $$ref{$option}) + { + my $modvalue = $$ref{$option}; + $value .= " $modvalue"; + } + + return $value; + } + + # As always qt-copy has to be difficult + if ($module eq 'qt-copy' and $option eq 'configure-flags') + { + return $defaultQtCopyArgs if not defined $$ref{$option}; + return $$ref{$option}; + } + + # Everything else overrides the global, unless of course it's not set. + # If we're reading for global options, we're pretty much done. + return $$ref{$option} if defined $$ref{$option}; + return ${$global_opts}{$option}; +} + +# Subroutine used to handle the checkout-only option. It handles +# updating subdirectories of an already-checked-out module. +# First parameter is the module, all remaining parameters are subdirectories +# to check out. +# +# Returns 0 on success, non-zero on failure. +sub update_module_subdirectories +{ + my $module = shift; + my $result; + + # If we have elements in @path, download them now + for my $dir (@_) + { + info "\tUpdating g[$dir]"; + $result = run_svn($module, "svn-up-$dir", [ 'svn', 'up', $dir ]); + return $result if $result; + } + + return 0; +} + +# Returns true if a module has a base component to their name (e.g. KDE/, +# extragear/, or playground). Note that modules that aren't in trunk/KDE +# don't necessary meet this criteria (e.g. kdereview is a module itself). +sub has_base_module +{ + my $module = shift; + + return $module =~ /^(extragear|playground|KDE)(\/[^\/]+)?$/; +} + +# Subroutine to return the directory that a module will be stored in. +# NOTE: The return value is a hash. The key 'module' will return the final +# module name, the key 'path' will return the full path to the module. The +# key 'fullpath' will return their concatenation. +# For example, with $module == 'KDE/kdelibs', and no change in the dest-dir +# option, you'd get something like: +# { +# 'path' => '/home/user/kdesvn/KDE', +# 'module' => 'kdelibs', +# 'fullpath' => '/home/user/kdesvn/KDE/kdelibs' +# } +# If dest-dir were changed to e.g. extragear-multimedia, you'd get: +# { +# 'path' => '/home/user/kdesvn', +# 'module' => 'extragear-multimedia', +# 'fullpath' => '/home/user/kdesvn/extragear-multimedia' +# } +# First parameter is the module. +# Second parameter is either source or build. +sub get_module_path_dir +{ + my $module = shift; + my $type = shift; + my $destdir = get_dest_dir($module); + my $srcbase = get_kdesvn_dir(); + $srcbase = get_build_dir($module) if $type eq 'build'; + + my $combined = "$srcbase/$destdir"; + + # Remove dup // + $combined =~ s/\/+/\//; + + my @parts = split(/\//, $combined); + my %result = (); + $result{'module'} = pop @parts; + $result{'path'} = join('/', @parts); + $result{'fullpath'} = "$result{path}/$result{module}"; + + return %result; +} + +sub get_fullpath +{ + my ($module, $type) = @_; + my %pathinfo = get_module_path_dir($module, $type); + + return $pathinfo{'fullpath'}; +} + +# Checkout a module that has not been checked out before, along with any +# subdirectories the user desires. +# The first parameter is the module to checkout (including extragear and +# playground modules), all remaining parameters are subdirectories of the +# module to checkout. +# Returns 0 on success, non-zero on failure. +sub checkout_module_path +{ + my ($module, @path) = @_; + my %pathinfo = get_module_path_dir($module, 'source'); + my $result; + my @args; + + if (not -e $pathinfo{'path'} and not super_mkdir($pathinfo{'path'})) + { + error "Unable to create path r[$pathinfo{path}]!"; + return 1; + } + + chdir($pathinfo{'path'}); + + push @args, ('svn', 'co'); + push @args, '-N' if scalar @path; + push @args, svn_module_url($module); + push @args, $pathinfo{'module'}; + + note "Checking out g[$module]"; + $result = run_svn($module, 'svn-co', \@args); + return $result if $result; + + chdir($pathinfo{'module'}) if scalar @path; + + return update_module_subdirectories($module, @path); +} + +# Update a module that has already been checked out, along with any +# subdirectories the user desires. +# The first parameter is the module to checkout (including extragear and +# playground modules), all remaining parameters are subdirectories of the +# module to checkout. +# Returns 0 on success, non-zero on failure. +sub update_module_path +{ + my ($module, @path) = @_; + my $fullpath = get_fullpath($module, 'source'); + my $result; + my @args; + + chdir $fullpath; + + push @args, ('svn', 'up'); + push @args, '-N' if scalar @path; + + note "Updating g[$module]"; + + $result = run_svn($module, 'svn-up', \@args); + + if($result) # Update failed, try svn cleanup. + { + info "\tUpdate failed, trying a cleanup."; + $result = safe_system('svn', 'cleanup'); + + return $result if $result; + + info "\tCleanup complete."; + # Now try again. + + $result = run_svn($module, 'svn-up-2', \@args); + } + + return $result if $result; + + # If the admin dir exists and is a soft link, remove it so that svn can + # update it if need be. The link will automatically be re-created later + # in the process if necessary by the build functions. + unlink ("$fullpath/admin") if -l "$fullpath/admin"; + + return update_module_subdirectories($module, @path); +} + +# Subroutine to run a command with redirected STDOUT and STDERR. First parameter +# is name of the log file (relative to the log directory), and the +# second parameter is a reference to an array with the command and +# its arguments +sub log_command +{ + my $pid; + my $module = shift; + my $filename = shift; + my @command = @{(shift)}; + my $logdir = get_log_dir($module); + + debug "log_command(): Module $module, Command: ", join(' ', @command); + + if (pretending) + { + pretend "\tWould have run g[", join (' ', @command); + return 0; + } + + if ($pid = fork) + { + # Parent + waitpid $pid, 0; + + # If the module fails building, set an internal flag in the module + # options with the name of the log file containing the error message. + my $result = $?; + set_error_logfile($module, "$filename.log") if $result; + + # If we are using the alias to a kdesvn-build function, it should have + # already printed the error message, so clear out errno (but still + # return failure status). + if ($command[0] eq 'kdesvn-build') + { + $! = 0; + } + + return $result; + } + else + { + # Child + if (not defined $logdir or not -e $logdir) + { + # Error creating directory for some reason. + error "\tLogging to std out due to failure creating log dir."; + } + + # Redirect stdout and stderr to the given file. + if (not debugging) + { +# Comment this out because it conflicts with make-install-prefix +# open (STDIN, "$logdir/$filename.log") or do { + error "Error opening $logdir/$filename.log for logfile."; + # Don't abort, hopefully STDOUT still works. + }; + } + else + { + open (STDOUT, "|tee $logdir/$filename.log") or do { + error "Error opening pipe to tee command."; + # Don't abort, hopefully STDOUT still works. + }; + } + + # Make sure we log everything. If the command is svn, it is possible + # that the client will produce output trying to get a password, so + # don't redirect stderr in that case. + open (STDERR, ">&STDOUT") unless $command[0] eq 'svn'; + + # Call internal function, name given by $command[1] + if($command[0] eq 'kdesvn-build') + { + debug "Calling $command[1]"; + + my $cmd = $command[1]; + splice (@command, 0, 2); # Remove first two elements. + + no strict 'refs'; # Disable restriction on symbolic subroutines. + if (not &{$cmd}(@command)) # Call sub + { + exit EINVAL; + } + + exit 0; + } + + # External command. + exec (@command) or do { + my $cmd_string = join(' ', @command); + error <) + { + chomp; + + # Update terminal (\e[K clears the line) if the percentage + # changed. + if (/([0-9]+)% (creating|compiling|linking)/) + { + print STDERR "\r$1% \e[K" unless ($1 == $last); + $last = $1; + } + } + + close(CHILD); + print STDERR "\r\e[K"; + + # If the module fails building, set an internal flag in the module + # options with the name of the log file containing the error message. + my $result = $?; + set_error_logfile($module, "$filename.log") if $result; + + return $result; + } + else + { + # Child + if (not defined $logdir or not -e $logdir) + { + # Error creating directory for some reason. + error "\tLogging to standard output due to failure creating log dir."; + } + + open (STDOUT, "|tee $logdir/$filename.log") or do { + error "Error opening pipe to tee command." + }; + + # Make sure we log everything. + open (STDERR, ">&STDOUT"); + + exec (@command) or do { + my $cmd_string = join(' ', @command); + error < q(-system-zlib -qt-gif -system-libjpeg -system-libpng + -plugin-imgfmt-mng -thread -no-exceptions -debug + -dlopen-opengl), + 'apply-qt-patches' => 'true', + +# See setup_kde35_hack() for why this option is here. + 'module-base-path' => 'branches/qt/3.3', + + 'use-qt-builddir-hack' => 'true', + 'use-unsermake' => 0, + 'set-env' => { }, + }; + + # That handy q() construct above kept the newlines, I don't want them. + $package_opts{'qt-copy'}{'conf-flags'} =~ s/\s+/ /gm; +} + +# Reads in the options from the config file and adds them to the option store. +# The first parameter is a reference to the file handle to read from. +# The second parameter is 'global' if we're reading the global section, or +# 'module' if we should expect an end module statement. +sub parse_module +{ + my ($fh, $module) = @_; + $module = 'global' unless $module; + + # Make sure we acknowledge that we read the module name in from the + # file. + if (not defined $package_opts{$module}) + { + $package_opts{$module} = { + 'set-env' => { } + }; + } + + # Read in each option + while (<$fh>) + { + # Handle line continuation + chomp; + + if(s/\\\s*$//) # Replace \ followed by optional space at EOL and try again. + { + $_ .= <$fh>; + redo unless eof($fh); + } + + s/#.*$//; # Remove comments + next if /^\s*$/; # Skip blank lines + + if($module eq 'global') + { + last if /^end\s+global/; # Stop + } + else + { + last if /^end\s+module/; # Stop + } + + # The option is the first word, followed by the + # flags on the rest of the line. The interpretation + # of the flags is dependant on the option. + my ($option, $value) = /^\s* # Find all spaces + ([-\w]+) # First match, alphanumeric, -, and _ + # (?: ) means non-capturing group, so (.*) is $value + # So, skip spaces and pick up the rest of the line. + (?:\s+(.*))?$/x; + + $value = "" unless defined $value; + + # Simplify this. + $value =~ s/\s+$//; + $value =~ s/^\s+//; + $value =~ s/\s+/ /; + + # Check for false keyword and convert it to Perl false. + $value = 0 if lc($value) =~ /^false$/; + + # Replace tildes with home directory. + 1 while ($value =~ s"(^|:|=)~/"$1$ENV{'HOME'}/"); + + set_option($module, $option, $value); + } +} + +# This subroutine reads in the settings from the user's configuration +# file. +sub read_options +{ + # The options are stored in the file $rcfile + my $success = 0; + my $global_opts = $package_opts{'global'}; + for my $file (@rcfiles) + { + if (open CONFIG, "<$file") + { + $success = 1; + $rcfile = $file; + last; + } + } + + if (not $success) + { + if(scalar @rcfiles == 1) + { + # This can only happen if the user uses --rc-file, if we fail to + # load the file, we need to fail to load. + error <) + { + s/#.*$//; # Remove comments + next if (/^\s*$/); # Skip blank lines + + # First command in .kdesvn-buildrc should be a global + # options declaration, even if none are defined. + if (not /^global\s*$/) + { + error "Invalid configuration file: $rcfile."; + error "Expecting global settings section!"; + exit 1; + } + + # Now read in each global option + parse_module(\*CONFIG, 'global'); + last; + } + + my $using_default = 1; + + # Now read in module settings + while () + { + s/#.*$//; # Remove comments + next if (/^\s*$/); # Skip blank lines + + # Get modulename (has dash, dots, slashes, or letters/numbers) + ($modulename) = /^module\s+([-\/\.\w]+)\s*$/; + + if (not $modulename) + { + warning "Invalid configuration file $rcfile!"; + warning "Expecting a start of module section."; + warning "Global settings will be retained."; + + $modulename = 'null'; # Keep reading the module section though. + } + + # Don't build default modules if user has their own wishes. + if ($using_default) + { + $using_default = 0; + @update_list = @build_list = ( ); + } + + parse_module(\*CONFIG, $modulename); + + next if ($modulename eq 'null'); + + # Done reading options, add this module to the update list + push (@update_list, $modulename) unless exists $ignore_list{$modulename}; + + # Add it to the build list, unless the build is only + # supposed to be done manually. + if (not get_option ($modulename, 'manual-build') and not exists $ignore_list{$modulename}) + { + push (@build_list, $modulename); + } + } + + close CONFIG; + + delete $package_opts{'null'}; # Just in case. + + # For the 3.5 edition we want to set the qt-copy option module-base-path + # to branches/qt/3.3 unless the user already has it set. + unless (exists $package_opts{'qt-copy'}{'module-base-path'}) + { + set_option ('qt-copy', 'module-base-path', 'branches/qt/3.3'); + } + + # If the user doesn't ask to build any modules, build a default set. + # The good question is what exactly should be built, but oh well. + setup_default_modules() if $using_default; +} + +# Subroutine to check if the given module needs special treatment to support +# srcdir != builddir. If this function returns true kdesvn-build will use a +# few hacks to simulate it, and will update e.g. configure paths appropriately +# as well. +sub module_needs_builddir_help +{ + my $module = shift; + my @module_help_list = qw/qt-copy kdebindings valgrind/; + + # qt-copy special case to support use-qt-builddir-hack. + if ($module eq 'qt-copy' and not get_option('qt-copy', 'use-qt-builddir-hack')) + { + return 0; + } + + return list_has(@module_help_list, $module); +} + +# This subroutine reads the set-env option for a given module and initializes +# the environment based on that setting. +sub setup_module_environment +{ + my $module = shift; + my ($key, $value); + + # Let's see if the user has set env vars to be set. + my $env_hash_ref = get_option($module, 'set-env'); + while (($key, $value) = each %{$env_hash_ref}) + { + setenv($key, $value); + } +} + +# Subroutine to initialize some environment variable for building +# KDE from Subversion. Change this section if a dependency changes later. +sub initialize_environment +{ + $ENV{"WANT_AUTOMAKE"} = "1.7"; + $ENV{"WANT_AUTOCONF_2_5"} = "1"; + $ENV{"PATH"} = get_option ('global', 'binpath'); + + my $svnserver = get_option ('global', 'svn-server'); + + my $pc_path = get_option('global', 'kdedir') . "/lib/pkgconfig"; + $pc_path .= ":" . $ENV{'PKG_CONFIG_PATH'} if ( exists $ENV{'PKG_CONFIG_PATH'} ); + $ENV{'PKG_CONFIG_PATH'} = $pc_path; + + if(-t STDOUT and get_option('global', 'colorful-output')) + { + $RED = "\e[31m"; + $GREEN = "\e[32m"; + $YELLOW = "\e[33m"; + $NORMAL = "\e[0m"; + $BOLD = "\e[1m"; + } + + # Set the process priority + setpriority PRIO_PROCESS, 0, get_option('global', 'niceness'); + + setup_module_environment ('global'); +} + +# Subroutine to get a list of modules to install, either from the command line +# if it's not empty, or based on the list of modules successfully built. +sub get_install_list +{ + my @install_list; + + if ($#ARGV > -1) + { + @install_list = @ARGV; + @ARGV = (); + } + else + { + # Get list of built items from $logdir/latest/build-status + my $logdir = get_subdir_path('global', 'log-dir'); + + if (not open BUILTLIST, "<$logdir/latest/build-status") + { + error "Can't determine what modules have built. You must"; + error "specify explicitly on the command line what modules to build."; + exit (1); # Don't finish, no lock has been taken. + } + + while () + { + chomp; + if (/Succeeded/) + { + # Clip to everything before the first colon. + my $module = (split(/:/))[0]; + push @install_list, $module; + } + } + + close BUILTLIST; + } + + return @install_list; +} + +# Print out an error message, and a list of modules that match that error +# message. It will also display the log file name if one can be determined. +# The message will be displayed all in uppercase, with PACKAGES prepended, so +# all you have to do is give a descriptive message of what this list of +# packages failed at doing. +sub output_failed_module_list($@) +{ + my ($message, @fail_list) = @_; + $message = uc $message; # Be annoying + + debug "Message is $message"; + debug "\tfor ", join(', ', @fail_list); + + if (scalar @fail_list > 0) + { + my $homedir = $ENV{'HOME'}; + my $logfile; + + warning "\nr[b[<<< PACKAGES $message >>>]"; + + for (@fail_list) + { + $logfile = get_option($_, '#error-log-file'); + $logfile = "No log file" unless $logfile; + $logfile =~ s|$homedir|~|; + + warning "r[$_] - g[$logfile]"; + } + } +} + +# This subroutine reads the fail_lists dictionary to automatically call +# output_failed_module_list for all the module failures in one function +# call. +sub output_failed_module_lists() +{ + for my $type (@fail_display_order) + { + my @failures = @{$fail_lists{$type}}; + output_failed_module_list("failed to $type", @failures); + } +} + +# This subroutine extract the value from options of the form --option=value, +# which can also be expressed as --option value. The first parameter is the +# option that the user passed to the cmd line (e.g. --prefix=/opt/foo), and +# the second parameter is a reference to the list of command line options. +# The return value is the value of the option (the list might be shorter by +# 1, copy it if you don't want it to change), or undef if no value was +# provided. +sub extract_option_value($\@) +{ + my ($option, $options_ref) = @_; + + if ($option =~ /=/) + { + my @value = split(/=/, $option); + shift @value; # We don't need the first one, that the --option part. + + return undef if (scalar @value == 0); + + # If we have more than one element left in @value it's because the + # option itself has an = in it, make sure it goes back in the answer. + return join('=', @value); + } + + return undef if scalar @{$options_ref} == 0; + return shift @{$options_ref}; +} + +# Utility subroutine to handle setting the environment variable type of value. +# Returns true (non-zero) if this subroutine handled everything, 0 otherwise. +# The first parameter should by the reference to the hash with the 'set-env' +# hash ref, second parameter is the exact option to check, and the third +# option is the value to set that option to. +sub handle_set_env +{ + my ($href, $option, $value) = @_; + + return 0 if $option !~ /^#?set-env$/; + + my ($var, @values) = split(' ', $value); + + $$href{$option} = ( ) unless exists $$href{$option}; + $$href{$option}{$var} = join(' ', @values); + + return 1; +} + +# Sets the option for the given module to the given value. If the data for the +# module doesn't exist yet, it will be defined starting with a default value. +# First parameter: module to set option for (or 'global') +# Second parameter: option name (Preceded by # for a sticky option) +# Third parameter: option value +# Return value is void +sub set_option +{ + my ($module, $option, $value) = @_; + + # Set module options + if (not exists $package_opts{$module}) + { + $package_opts{$module} = { + 'set-env' => { } + }; + } + + return if handle_set_env($package_opts{$module}, $option, $value); + $package_opts{$module}{$option} = $value; +} + +# Subroutine to process the command line arguments. Any arguments so +# processed will be removed from @ARGV. +# The arguments are generally documented in doc.html now. +# NOTE: Don't call finish() from this routine, the lock hasn't been obtained. +# NOTE: The options have not been loaded yet either. Any option which +# requires more than rudimentary processing should set a flag for later work. +sub process_arguments +{ + my $arg; + my $version = "kdesvn-build 0.97.6 (KDE 3.5 Edition)"; + my $author = < + +Many people have contributed code, bugfixes, and documentation. + +Please report bugs using the KDE Bugzilla, at http://bugs.kde.org/ +DONE + + my @argv; + + while ($_ = shift @ARGV) + { + SWITCH: { + /^(--version)$/ && do { print "$version\n"; exit; }; + /^--author$/ && do { print $author; exit; }; + /^(-h)|(--?help)$/ && do { + print < Read configuration from filename instead of default. + --nice= Allows you to run the script with a lower priority + The default value is 10 (lower priority by 10 steps). + --prefix=/kde/path This option is a shortcut to change the setting for + kdedir from the command line. It implies + --reconfigure. + + --resume Tries to resume the make process from the last time + the script was run, without performing the Subversion + update. + --resume-from= Starts building from the given package, without + performing the Subversion update. + --revision (or -r)= Forces update to revision from Subversion. + + --refresh-build Start the build from scratch. + --reconfigure Run configure again, but don't clean the build + directory or re-run make -f Makefile.cvs. + --recreate-configure Run make -f Makefile.cvs again to redo the configure + script. + --no-rebuild-on-fail Don't try to rebuild a module from scratch if it + failed building and we didn't already try to build it + from scratch. + --build-system-only Create the build infrastructure, but don't actually + perform the build. + --install Try to install the packages passed on the command + line, or all packages in ~/.kdesvn-buildrc that don't + have manual-build set. Building and Subversion + updates are not performed. + + --