Update 2.9 to 2.10
This update is proposed as
  http://mail-index.netbsd.org/tech-pkg/2014/12/08/msg013888.html
- (revbump) convert internal to two pass, and reduce fallout
- (replace-commonbump) new command to replace 'commonbump' to
  'commonbump.replaced'
- (check-cvs-diff) new command to check 'cvs diff -u' if py27,py34, ruby200
  and replace it if exists
  Thanks wiz@ and gdt@ for advice.


(mef)

#!@PERL@ -w
# -*- perl -*-
#
# read output of (cd /usr/pkgsrc; cvs diff -u ) and check if
# py{27,34} and ruby200 string are left in buildlink3.mk.
# They are then modified to Variable representation.
#"
use strict;
use Getopt::Std;
our(@ARGV);
my (%opts);

my($my_name) = $0;
$my_name =~ s#(.*)/##;

sub usage(){
    print <<HELP;
$my_name: 
    Read cvs diff -u output (of named file) and substitute
    if the string py27, py34 or ruby200 is found in the corresponding
    changes in diff.
Synopsys:
    $my_name  [-h] [-p pkgsrc_dir] [-v] [cvs-diff-u_output-file]
Where:
    -h  Show this help
    -p  pkgsrc directory other than /usr/pkgsrc
    -v  verbose (not actually implemented yet)
    cvs-diff-u_output_file:
        The name of input. Usually the file including 'cvs diff -u' 
        output (default cvs-diff) 
See Also:
    revbump(1) for how to use it.
HELP
}    

sub main() {
    my ($PKGSRCDIR) =  '/usr/pkgsrc';
    my ($CVS_DIFF)  = 'cvs-diff';
    my ($file_to_edit);
    my ($stay, $mod) = (0,0);
    my ($Makefile, $buildlink3, $other) = (0, 0, 0);

    getopts('hvp:',\%opts);
    if ($ARGV[0])   { $CVS_DIFF = $ARGV[0]}
    if ($opts{'p'}) { $PKGSRCDIR = $opts{'p'}; }
    if ($opts{'h'}) { usage() ; exit ;}

   # for using rename, unlink 
    chdir $PKGSRCDIR;
    
    open(CVS_DIFF, $CVS_DIFF) || die "Problem opening file $CVS_DIFF: $!\n";
    while(<CVS_DIFF>){
	# Looking for the +++ filename line in cvs diff (supposed to use 'cvs diff -u')
	$file_to_edit = '';
	if ( /^\+\+\+ (\S+)/) {
	    $file_to_edit = $1; }
	if ( $file_to_edit && -f $file_to_edit ) {
	    if    ( $file_to_edit =~ /Makefile$/ )      { $Makefile++;}
	    elsif ( $file_to_edit =~ /buildlink3.mk$/ ) { $buildlink3++;
		#	    print __LINE__, ' ', $file_to_edit,"\n";
		my ($new_file) = $file_to_edit. '.new';
		my ($edit) = 0;
		open(NEW,  "> $new_file" ) || print STDERR "Problem to write $edit: $! \n";
		open(EDIT, $file_to_edit ) || print STDERR "Problem opening file $file_to_edit: $! \n";
		while(<EDIT>) {
		    if ( /^BUILDLINK.*py27/ )    {  $_=~    s/py27/\${PYPKGPREFIX}/   ; $edit++;}
    		    if ( /^BUILDLINK.*py34/ )    {  $_=~    s/py34/\${PYPKGPREFIX}/   ;
						    print STDERR "   py34 found at $file_to_edit\n";
						    ; $edit++;}
		    if ( /^BUILDLINK.*ruby200/ ) {  $_=~ s/ruby200/\${RUBY_PKGPREFIX}/ ; $edit++;}
		    print NEW $_;
		}
		close(EDIT);
		close(NEW);
		if ($edit) { unlink $file_to_edit; 
			     rename $new_file, $file_to_edit;
			     $mod++;}
		else {
		    $stay++;
		    unlink $new_file; };
	    } else  { $other++; print STDERR "  (other) ", $file_to_edit,"\n"}
	    }
        }
	close(CVS_DIFF);
    printf STDERR "Makefile:      %4d\n", $Makefile;
    printf STDERR "buildlink3.mk: %4d\n", $buildlink3;
    printf STDERR "  Modified:    %4d\n", $mod;
    printf STDERR "  Untouched:   %4d\n", $stay;
    printf STDERR "Other:         %4d\n", $other;
}

main();
exit;

.\"	$NetBSD: check-cvs-diff.1,v 1.1 2014/12/15 23:25:54 mef Exp $
.\"
.\" Copyright (c) 2003, 2004, 2005 The NetBSD Foundation, Inc.
.\"
.\" This code was originally contributed to the NetBSD Foundation, Inc.
.\" by Makoto Fujiwara <mef@NetBSD.org>.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\"    notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\"    notice, this list of conditions and the following disclaimer in
.\"    the documentation and/or other materials provided with the
.\"    distribution.
.\" 3. Neither the name of author nor the names of its contributors may
.\"    be used to endorse or promote products derived from this software
.\"    without specific prior written permission.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
.\" CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
.\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
.\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
.\" IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
.\" GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
.\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
.\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
.\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.\"
.Dd December 14, 2014    
.Dt CHECK-CVS-DIFF 1
.Os
.Sh NAME
.Nm check-cvs-diff
.Nd tool to check if revbump process described in revbump(1) results including py27, py34 or ruby200
in the files. And then replaces the strings by the variable notation.
.Sh SYNOPSIS
.Nm
.Op Fl h
.Op Fl p Ar pkgsrcdir
.Op Fl v
cvs-diff-u_output-file
.Sh DESCRIPTION
.Nm
is a perl script for pkgsrc developers that simplifies checking and replacing fixed
version number in revbump resultant tree, Makefile and buildlink3.mk files.
It reads `cvs diff -u' output written in named file, and if above mentioned strings found in the diff, replaces it in corresponding file by apropriate variable notation.
.Pp
.Nm Fl h
displays a short usage.
.Pp
If
.Fl p
is given,
.Ar pkgsrcdir
is used instead of the default
.Pa /usr/pkgsrc .
.Pp
If
.Fl v
is given,
it outputs some verbose information.

.Nm
prints the package name
(in
.Ar category/package
form) on stdout for each package where it failed, but usually none.

.Sh SEE ALSO
.Xr revbump 1 ,
.Xr finddepends 1 ,
.Xr replace-commonbump 1

#!@PERL@ -w
# -*- perl -*-
# The process has two steps
# (1)
# Travers all the files (with some filtering) over /usr/pkgsrc
# pick up referenced -> referer relation and store them to %HASH 
#  (multiple files are concatinated by ';' )
# (2)
# read input (usually named 'commonbump'), and expand one line into
# replacement referer lines, which are referenced  -> referer relation.

use strict;
use Getopt::Std;
use File::Find;

my(%HASH);	  		# referenced -> referer relation hash, referer is delimitted by ;
my($PKGSRCDIR) = '/usr/pkgsrc';	# /usr/pkgsrc path, usually /usr/pkgsrc

my($TABLE) = 'commonbump';	# name of file, list-of-files to process
my(@TABLE);			# contents of file above

my($debug) = 0;
my($prefix) ;			# A part of path, top to pkgsrc, typically /usr/pkgsrc

my($common_bump) = 0;		# count, not used for now (yet)
my(%opts);
our(@ARGV);

my ($FILE)	= __FILE__;
    $FILE	=~ s,.*/([^/]+),$1,;	# pick leafname from __FILE__

my($replaced) = 'commonbump.replaced';

my($my_name) = $0;
$my_name =~ s#(.*)/##;

sub usage() {
    print <<HELP;
$my_name:
  This command is a part of revbump package and intended for pkgsrc developers.
  (1) Read entire pkgsrc tree and check referer -> referenced relation,
      then internally keeps (opposite) referenced -> referer relation table.
  (2) Read another file, named '$TABLE' or other name with -T, which
      usually lists Makefile.common or *common.mk files to expand.
  (3) Now expands those list in '$TABLE' into list of Makefiles by
      referenced -> refer relation. By default, it outputs to the file
      '$replaced' (currently the name is fixed).

Synopsys:
    $my_name [-a ] [-h] [-p pkgsrc_directory] [-T list_file] 
Where:
   -a	Check all, disregard to -T option. If "# used by" is 
    	included or not.
   -h	Show this help
   -p directory	
        pkgsrc directory to process (default $PKGSRCDIR)
   -T	the filename containing list of files (default commonbump)
See Also:
    revbump(1) for how to use it.
HELP
}

#  get value of variable by using make show-var
sub show_var($$){
    my($varname) = shift;
    my($pkgdir)  = shift;
    my($value);
    #chdir $pkgdir;
    open(MAKE, '-|', "cd $PKGSRCDIR/$pkgdir ; make show-var VARNAME=$varname");
    $value = <MAKE>;
    close(MAKE);
    chomp($value);
    if ($value =~ /know how to make/ ) { print STDERR $value, '(', $pkgdir,')',"\n";}
    return $value
}

sub GenerateHash ($$) {
    my ($pkgsrc) = shift;
    my ($prefix) = shift;

    # ----------------------------------------------------------------
    my ($wanted_closure) = sub () {
	my $dir   = $File::Find::dir ;
	my $fname = "$File::Find::dir/$_" ;

	# skip these directories and files ( as a files including something )
	if ($dir =~ m|CVS$| )		{$File::Find::prune = 1; return;}
	if ($dir =~ m|mk$| )		{$File::Find::prune = 1; return;}
	if ($dir =~ m|work$| )		{$File::Find::prune = 1; return;}
	if ($dir =~ m|patches$| )	{$File::Find::prune = 1; return;}
	if ($dir =~ m|files$| )		{$File::Find::prune = 1; return;}
	if ($dir =~ m|bootstrap$| )	{$File::Find::prune = 1; return;}
	if ($dir =~ m|pkgsrc/[^/]$| )	{$File::Find::prune = 1; return;}
	if ($dir =~ m|x11-links| ){$File::Find::prune = 1; return;}	
	if ($_ =~ m|^\.\#| )		{                        return;}
	if ($_ =~ m|~$| )		{                        return;}
	if ($_ =~ m|buildlink3.mk$| )	{                        return;}
	if ($_ =~ m|builtin.mk$| )	{                        return;}

	my($shortname) = $fname;
	$shortname =~ s|$prefix/||;
	my($shortdir)  = $dir;
	$shortdir  =~ s|$prefix/||;

	# pick only Makefile.* and .mk  ( as a files including something )
	if ($_ =~ m|Makefile| ||
	    $_ =~ m|\.mk$|          )	{
	    print STDERR __LINE__, ' ', $fname,' ', `pwd`, "\n" if $debug;
	    open(FNAME, $fname) || print STDERR __LINE__, " Problem opening file $fname:$!\n";
	    my ($included) = '';
	    while (<FNAME>){
		# now starts finding included file		
		# pick .include "../../
		if ( m|^\.\s*include\s+\"\.\./\.\./(.*)\"| ) { $included = $1;}
		# pick ".include "Makefile" etc (without leading ../../), this needs to add package dir.
		if ( m|^\.\s*include\s+\"(.*)\"|           ) { next;}	# including the same directory stuff, doesn't matter
		if ( m|^\.\s*include\s+\"(.*)\"|           ) { $included = $prefix .'/'. $1;}		
		if ( $included =~ m|/mk/| )       	{ next;}	# it is include line but for mk, skip this line
		if ( $included =~ m|version.mk| )	{ next;}
		if ( $included =~ m|tests/| )		{ next;}
		if ( $included =~ m|options.mk| )	{ next;}
		if ( $included =~ m|enigmail.mk| )	{ next;}		

		$included =~ s,\${.CURDIR},$shortname,;
		if ($included =~ /\${([^}]+)}/ ) { 
		    my($varname) = $1;
		    my($value) = show_var($varname, $shortdir);
		    $included =~ s/\${[^}]+}/$value/;
		    if ($value eq '') {
			print STDERR 
$FILE, ': ', __LINE__, ' Value ${', $varname, '} is emtpy at ', $shortdir, "\n";
		    }
		}
		print STDERR __LINE__ , ' ', $shortname, ' -> ', $included,': ',$_ ,"\n" if $debug;
		if ( $included eq '') 	   		{ next;}	# not include line, look at next line
		print STDERR __LINE__ , ' ', $shortname, ' -> ', $included,"\n" if $debug;
		if ( ! $HASH{$included} ) {
		    $HASH{$included} =  $shortname;
		} else {
		    my(@registered) = split ';',  $HASH{$included};
		    if (grep (/^$shortname$/, @registered) == 0 ) {
			$HASH{$included} .= ';'. $shortname;
			}
		}
	    } # end while
	    close(FNAME);
	} # if of (major process) ... starting with: if ($_ =~ m|Makefile| || 
    };  # end of my ($wanted_closure) = sub () {
    # ----------------------------------------------------------------    
    find($wanted_closure, $pkgsrc);
}

# Table is assumed to contain list of files to process, set up it in @TABLE here.
sub ReadTable($) {
    my ($table) = shift;
    if ($table eq '-') {
	@TABLE = <>;
    } else {
	open(TABLE, $table) || print STDERR "Problem reading file $table: $!\n";
	@TABLE = <TABLE>;
	close(TABLE);
	}
    }

sub ShowResults() {
    open(REPLACED, "> $replaced") || die "problem open to write: $replaced: $!\n";
    foreach my $file (@TABLE) {
	chomp($file);
	if ($file =~ /^\s*$/  ) { next; }	# Skip empty line, in case
	if ($HASH{$file} && 			# To avoid 'Use of uninitialized value in split at ..'
	    grep ($file, $HASH{$file}) > 0) {	# referer found ( separated with ';')
	    my(@list) = split ';', $HASH{$file};
	    foreach my $i (0..$#list) {	    
		print REPLACED "$list[$i]\n";	# write the list into REPLACED handle
	    }
	}
    }
}

# check the pkgsrc tree is healthy or not
sub CheckPkgsrcTree($){
    my ($pkgsrc) =	shift;
    
    if (! -d $pkgsrc || ! -d "$pkgsrc/doc" || ! -d "$pkgsrc/mk") {
	print STDERR "Invalid pkgsrc directory $pkgsrc\n";
	exit 1;
    }
}

# if -a option is applied, not using @TABLE, but scan whole thing.
sub CheckAll() {
    foreach my $i (sort keys %HASH) {	# for all the referenced
	if ($i =~ /buildlink3.mk/ ) { next;}

	# First collect the line of '# used by .*'
	open (REFERENCED, $i) || print '  *** ', __LINE__, " Unable to open $i $!\n";
	my (@referer) = {};
	while(<REFERENCED>){
	    if (/\# used by (.*)/) { push (@referer, $1);}
	}
	close(REFERENCED);
	my $number = 0;
	foreach my $referer (@referer) {
	    $number += grep $referer, $HASH{$i};
	}
	$#referer++; 	# conpensate -1 -> 0, 0 -> 1 etc
    	if ($#referer <  $number) { print $#referer . ' ? ' . $number .' .. ', $i,' <- ', $HASH{$i}, ' ... <', "\n";}
    	if ($#referer >  $number) { print $#referer . ' ? ' . $number .' .. ', $i,' <- ', $HASH{$i}, ' ... >', "\n";}	
    }
}
sub main() {
    my($check_all) = 0;
    my($prefix);
    getopts('ahp:T:', \%opts);

    if ($opts{'a'}) { $check_all = 1;  }
    if ($opts{'h'}) { usage(); exit 0;}
    if ($opts{'p'}) { $PKGSRCDIR = $opts{'p'};}
    if ($opts{'T'}) { $TABLE = $opts{'T'};}

    $prefix = $PKGSRCDIR;
    $prefix =~ s|.*/pkgsrc/(.*)|$1|;
    CheckPkgsrcTree($PKGSRCDIR);
    GenerateHash($PKGSRCDIR, $prefix);
    ReadTable($TABLE);
    if ($check_all) { CheckAll(); }
    else            { ShowResults();}
}

main();

exit;
__END__

.\"	$NetBSD: replace-commonbump.1,v 1.1 2014/12/15 23:25:54 mef Exp $
.\"
.\" Copyright (c) 2014 The NetBSD Foundation, Inc.
.\"
.\" This code was originally contributed to the NetBSD Foundation, Inc.
.\" by Makoto Fujiwara <mef@NetBSD.org>.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\"    notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\"    notice, this list of conditions and the following disclaimer in
.\"    the documentation and/or other materials provided with the
.\"    distribution.
.\" 3. Neither the name of author nor the names of its contributors may
.\"    be used to endorse or promote products derived from this software
.\"    without specific prior written permission.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
.\" CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
.\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
.\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
.\" IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
.\" GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
.\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
.\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
.\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.\"
.Dd December 14, 2014
.Dt REPLACE-COMMONBUMP 1
.Os
.Sh NAME
.Nm replace-commonbump
.Nd tool to help expanding (replacing) commonbump file to the real list of Makefile's.
.Sh SYNOPSIS
.Nm
.Op Fl a
.Op Fl h
.Op Fl p Ar pkgsrcdir
.Op Fl T Ar list_file
.Sh DESCRIPTION
.Nm
is a perl script for pkgsrc developers. It simplifies the process expanding
`commonbump' file into `commonbump-replace'. Commonbump file is to be created by the first stage of revbump process, as is described in revbump(1).
.Pp
.Nm Fl h
displays a short usage.
.Pp
If
.Fl p
is given,
.Ar pkgsrcdir
is used instead of the default
.Pa /usr/pkgsrc .
.Pp
If
.Fl a
is given,
check all pkgsrc tree, disregard neither to
.Fl T
option nor to the default `commonbump' file.
It will check if "# used by" is included or not, usually in Makefile.common in the package.
.Pp
If
.Fl T
is given,
.Ar list_file
is used instead of the default
.Pa `commonbump' .
.Pp
.Nm
prints the package name
(in
.Ar category/package
form) on stdout for each package where it failed, but usually none.

The internal process is divided into three stages.

.item
(1)
Read entire pkgsrc tree and check a
.Ar referer -> referenced
relation,
then internally keeps (opposite)
.Ar referenced -> referer
relation table.

.item
(2) Read a file, named `commonbump' or the other name given with -T, which
usually lists Makefile.common or *common.mk files to be expanded.

.item
(3) Now expands the filename listed in `commonbump' into the list of packagename/Makefile by
referenced -> referer relation. It then outputs to the file
`commonbump.replaced' (currently the name is fixed).

.Sh SEE ALSO
.Xr revbump 1 ,
.Xr blbump 1 ,
.Xr check-cvs-diff 1 ,
.Xr finddepends 1