Wed Sep 11 05:25:55 2019 UTC ()
pkgtools/url2pkg: update to 2.34

Changes since 2.33:

* code cleanup in MakeMaker.pm
* don't create a PLIST for Perl packages

* allow Python dependencies from setup.py to use spaces around >=
* properly indented Python code according to PEP 8
* added mock for setuptools.find_package
* only try to migrate a package from GitHub to PyPI if it is really
  a Python package on GitHub


(rillig)
diff -r1.96 -r1.97 pkgsrc/pkgtools/url2pkg/Makefile
diff -r1.9 -r1.10 pkgsrc/pkgtools/url2pkg/files/MakeMaker.pm
diff -r1.1 -r1.2 pkgsrc/pkgtools/url2pkg/files/setuptools.py
diff -r1.64 -r1.65 pkgsrc/pkgtools/url2pkg/files/url2pkg.pl

cvs diff -r1.96 -r1.97 pkgsrc/pkgtools/url2pkg/Makefile (expand / switch to unified diff)

--- pkgsrc/pkgtools/url2pkg/Makefile 2019/09/09 08:08:02 1.96
+++ pkgsrc/pkgtools/url2pkg/Makefile 2019/09/11 05:25:55 1.97
@@ -1,16 +1,16 @@ @@ -1,16 +1,16 @@
1# $NetBSD: Makefile,v 1.96 2019/09/09 08:08:02 maya Exp $ 1# $NetBSD: Makefile,v 1.97 2019/09/11 05:25:55 rillig Exp $
2 2
3PKGNAME= url2pkg-2.33 3PKGNAME= url2pkg-2.34
4CATEGORIES= pkgtools 4CATEGORIES= pkgtools
5 5
6MAINTAINER= pkgsrc-users@NetBSD.org 6MAINTAINER= pkgsrc-users@NetBSD.org
7HOMEPAGE= http://www.NetBSD.org/docs/pkgsrc/creating.html 7HOMEPAGE= http://www.NetBSD.org/docs/pkgsrc/creating.html
8COMMENT= Tool to automate initial steps in building a package 8COMMENT= Tool to automate initial steps in building a package
9LICENSE= 2-clause-bsd 9LICENSE= 2-clause-bsd
10 10
11WRKSRC= ${WRKDIR} 11WRKSRC= ${WRKDIR}
12NO_CHECKSUM= yes 12NO_CHECKSUM= yes
13NO_BUILD= yes 13NO_BUILD= yes
14USE_LANGUAGES= # none 14USE_LANGUAGES= # none
15USE_TOOLS+= perl:run 15USE_TOOLS+= perl:run
16AUTO_MKDIRS= yes 16AUTO_MKDIRS= yes
@@ -19,27 +19,23 @@ do-extract: @@ -19,27 +19,23 @@ do-extract:
19 ${RUN} cd ${FILESDIR} && cp *.* ${WRKSRC}/ 19 ${RUN} cd ${FILESDIR} && cp *.* ${WRKSRC}/
20 20
21do-test: 21do-test:
22 ${RUN} cd ${WRKSRC} && env PKGSRCDIR=${PKGSRCDIR} perl -I. url2pkg.t 22 ${RUN} cd ${WRKSRC} && env PKGSRCDIR=${PKGSRCDIR} perl -I. url2pkg.t
23 23
24.include "../../mk/bsd.prefs.mk" 24.include "../../mk/bsd.prefs.mk"
25 25
26SUBST_CLASSES+= up 26SUBST_CLASSES+= up
27SUBST_STAGE.up= do-configure 27SUBST_STAGE.up= do-configure
28SUBST_MESSAGE.up= Replacing variable placeholders 28SUBST_MESSAGE.up= Replacing variable placeholders
29SUBST_FILES.up= url2pkg.pl MakeMaker.pm 29SUBST_FILES.up= url2pkg.pl MakeMaker.pm
30SUBST_VARS.up= MAKE PERL5 PYTHONBIN 30SUBST_VARS.up= MAKE PERL5 PYTHONBIN
31SUBST_SED.up+= -e 's,@LIBDIR@,${PREFIX}/lib/url2pkg,g' 31SUBST_SED.up+= -e 's,@LIBDIR@,${PREFIX}/lib/url2pkg,g'
32.if defined(BATCH) 32SUBST_SED.up+= -e 's,@PKGSRCDIR@,${BATCH:D/usr/pkgsrc:U${PKGSRCDIR}},g'
33SUBST_SED.up+= -e 's,@PKGSRCDIR@,/usr/pkgsrc,g' 
34.else 
35SUBST_VARS.up+= PKGSRCDIR 
36.endif 
37 33
38do-install: 34do-install:
39 ${INSTALL_SCRIPT} ${WRKSRC}/url2pkg.pl ${DESTDIR}${PREFIX}/bin/url2pkg 35 ${INSTALL_SCRIPT} ${WRKSRC}/url2pkg.pl ${DESTDIR}${PREFIX}/bin/url2pkg
40 ${INSTALL_MAN} ${FILESDIR}/url2pkg.8 ${DESTDIR}${PREFIX}/${PKGMANDIR}/man8 36 ${INSTALL_MAN} ${FILESDIR}/url2pkg.8 ${DESTDIR}${PREFIX}/${PKGMANDIR}/man8
41 ${INSTALL_DATA} ${WRKSRC}/MakeMaker.pm ${DESTDIR}${PREFIX}/lib/url2pkg/ExtUtils/ 37 ${INSTALL_DATA} ${WRKSRC}/MakeMaker.pm ${DESTDIR}${PREFIX}/lib/url2pkg/ExtUtils/
42 ${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/ 38 ${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/
43 39
44.include "../../lang/python/application.mk" 40.include "../../lang/python/application.mk"
45.include "../../mk/bsd.pkg.mk" 41.include "../../mk/bsd.pkg.mk"

cvs diff -r1.9 -r1.10 pkgsrc/pkgtools/url2pkg/files/MakeMaker.pm (expand / switch to unified diff)

--- pkgsrc/pkgtools/url2pkg/files/MakeMaker.pm 2019/08/18 11:26:33 1.9
+++ pkgsrc/pkgtools/url2pkg/files/MakeMaker.pm 2019/09/11 05:25:55 1.10
@@ -29,87 +29,86 @@ @@ -29,87 +29,86 @@
29 29
30# This is a drop-in replacement for the MakeMaker Perl module. Instead 30# This is a drop-in replacement for the MakeMaker Perl module. Instead
31# of generating a Makefile, it extracts the dependency information for 31# of generating a Makefile, it extracts the dependency information for
32# other Perl modules. It is used to simplify the generation of pkgsrc 32# other Perl modules. It is used to simplify the generation of pkgsrc
33# packages for Perl modules. 33# packages for Perl modules.
34 34
35package ExtUtils::MakeMaker; 35package ExtUtils::MakeMaker;
36 36
37require 5.013002; 37require 5.013002;
38 38
39use strict; 39use strict;
40use warnings; 40use warnings;
41 41
42use constant conf_pkgsrcdir => '@PKGSRCDIR@'; 42my $url2pkg_pkgsrcdir = '@PKGSRCDIR@';
43 43
44BEGIN { 44BEGIN {
45 use Exporter; 45 use Exporter;
46 use vars qw(@ISA @EXPORT); 46 use vars qw(@ISA @EXPORT);
47 @ISA = qw(Exporter); 47 @ISA = qw(Exporter);
48} 48}
49 49
50# From lib/perl5/5.18.0/ExtUtils/MakeMaker.pm 50# From lib/perl5/5.18.0/ExtUtils/MakeMaker.pm
51our $VERSION = '6.66'; 51our $VERSION = '6.66';
52 52
53our $Verbose = 0; # exported 53our $Verbose = 0; # exported
54our @EXPORT = qw(&WriteMakefile &prompt $Verbose $version); 54our @EXPORT = qw(&WriteMakefile &prompt $Verbose $version);
55our @EXPORT_OK = qw(&neatvalue &_sprintf562); 55our @EXPORT_OK = qw(&neatvalue &_sprintf562);
56 56
57# Finds and returns the category a given package lies in. 57# Finds and returns the category a given package lies in.
58# If the package does not exist, C<undef> is returned. 58# If the package does not exist, an empty string is returned.
59# If the package exists more than once, it is unspecified which 59# If the package exists more than once, it is unspecified which
60# of the categories is returned. 60# of the categories is returned.
61sub find_category($) { 61sub url2pkg_find_category($) {
62 my ($pkg) = @_; 62 my ($pkg) = @_;
63 my ($retval, $pkgsrcdir); 
64 63
65 opendir(D, conf_pkgsrcdir) or die; 64 opendir(D, $url2pkg_pkgsrcdir) or die;
66 foreach my $cat (readdir(D)) { 65 my @categories = readdir(D);
67 next if ($cat =~ qr"^\."); 66 closedir(D) or die;
 67
 68 foreach my $cat (@categories) {
 69 next if $cat =~ qr"^\.";
68 70
69 if (-f (conf_pkgsrcdir."/${cat}/${pkg}/Makefile")) { 71 if (-f "$url2pkg_pkgsrcdir/$cat/$pkg/Makefile") {
70 $retval = $cat; 72 return $cat;
71 } 73 }
72 } 74 }
73 closedir(D) or die; 75 return "";
74 return $retval; 
75} 76}
76 77
77sub writeDependency($$) { 78sub url2pkg_write_dependency($$) {
78 my ($dep, $ver) = @_; 79 my ($dep, $ver) = @_;
79 80
80 my $pkgbase = "p5-" . ($dep =~ s/::/-/gr); 81 my $pkgbase = "p5-$dep" =~ s/::/-/gr;
81 my $category = find_category($pkgbase); 82 my $category = url2pkg_find_category($pkgbase);
82 83
83 if (defined($category)) { 84 if ($category ne "") {
84 printf("DEPENDS\t%s>=%s:../../%s/%s\n", $pkgbase, $ver, $category, $pkgbase); 85 printf("DEPENDS\t%s>=%s:../../%s/%s\n", $pkgbase, $ver, $category, $pkgbase);
85 return; 86 return;
86 } 87 }
87 88
88 # If the package does not exist but the Perl module can be 89 # If the package does not exist but the Perl module can be loaded, assume
89 # loaded, assume that no extra dependency is needed. 90 # that it is a built-in module and no dependency declaration is needed.
90 return if eval("use $dep $ver; 1;"); 91 return if eval("use $dep $ver; 1;");
91 92
92 die("$0: ERROR: No pkgsrc package found for dependency ${dep}>=${ver}.\n$@\n"); 93 die("$0: ERROR: No pkgsrc package found for dependency $dep>=$ver.\n$@\n");
93} 94}
94 95
95sub WriteMakefile(%) { 96sub WriteMakefile(%) {
96 my (%options) = @_; 97 my (%options) = @_;
97 98
98 if (exists($options{"PREREQ_PM"})) { 99 my $deps = $options{"PREREQ_PM"} || {};
99 my $deps = $options{"PREREQ_PM"}; 100 foreach my $dep (sort(keys(%$deps))) {
100 foreach my $dep (sort(keys(%{$deps}))) { 101 url2pkg_write_dependency($dep, $deps->{$dep});
101 writeDependency($dep, $deps->{$dep}); 
102 } 
103 } 102 }
104} 103}
105 104
106sub prompt(@) { 105sub prompt(@) {
107 my ($message, $default) = @_; 106 my ($message, $default) = @_;
108 107
109 return $default || ""; 108 return $default || "";
110} 109}
111 110
112sub neatvalue { 111sub neatvalue {
113 return; 112 return;
114} 113}
115 114

cvs diff -r1.1 -r1.2 pkgsrc/pkgtools/url2pkg/files/Attic/setuptools.py (expand / switch to unified diff)

--- pkgsrc/pkgtools/url2pkg/files/Attic/setuptools.py 2019/08/18 11:26:33 1.1
+++ pkgsrc/pkgtools/url2pkg/files/Attic/setuptools.py 2019/09/11 05:25:55 1.2
@@ -1,15 +1,15 @@ @@ -1,15 +1,15 @@
1# $NetBSD: setuptools.py,v 1.1 2019/08/18 11:26:33 rillig Exp $ 1# $NetBSD: setuptools.py,v 1.2 2019/09/11 05:25:55 rillig Exp $
2 2
3def url2pkg_print_depends(keyword, depends): 3def url2pkg_print_depends(keyword, depends):
4 for dep in depends: 4 for dep in depends:
5 if '>' not in dep: 5 print('%s\t%s%s' % (keyword, dep.replace(' ', ''), '' if '>' in dep else '>=0'))
6 dep = dep + '>=0' 
7 print('%s\t%s' % (keyword, dep)) 
8 6
9def setup(**kwargs): 7def setup(**kwargs):
 8 if 'install_requires' in kwargs:
 9 url2pkg_print_depends('DEPENDS', kwargs['install_requires'])
10 10
11 if 'install_requires' in kwargs: 11 if 'tests_require' in kwargs:
12 url2pkg_print_depends('DEPENDS', kwargs['install_requires']) 12 url2pkg_print_depends('TEST_DEPENDS', kwargs['tests_require'])
13 13
14 if 'tests_require' in kwargs: 14def find_packages(where='.', exclude=(), include=('*',)):
15 url2pkg_print_depends('TEST_DEPENDS', kwargs['tests_require']) 15 return []

cvs diff -r1.64 -r1.65 pkgsrc/pkgtools/url2pkg/files/Attic/url2pkg.pl (expand / switch to unified diff)

--- pkgsrc/pkgtools/url2pkg/files/Attic/url2pkg.pl 2019/09/09 08:08:02 1.64
+++ pkgsrc/pkgtools/url2pkg/files/Attic/url2pkg.pl 2019/09/11 05:25:55 1.65
@@ -1,15 +1,15 @@ @@ -1,15 +1,15 @@
1#! @PERL5@ 1#! @PERL5@
2# $NetBSD: url2pkg.pl,v 1.64 2019/09/09 08:08:02 maya Exp $ 2# $NetBSD: url2pkg.pl,v 1.65 2019/09/11 05:25:55 rillig Exp $
3# 3#
4 4
5# Copyright (c) 2010 The NetBSD Foundation, Inc. 5# Copyright (c) 2010 The NetBSD Foundation, Inc.
6# All rights reserved. 6# All rights reserved.
7# 7#
8# This code is derived from software contributed to The NetBSD Foundation 8# This code is derived from software contributed to The NetBSD Foundation
9# by Roland Illig. 9# by Roland Illig.
10# 10#
11# Redistribution and use in source and binary forms, with or without 11# Redistribution and use in source and binary forms, with or without
12# modification, are permitted provided that the following conditions 12# modification, are permitted provided that the following conditions
13# are met: 13# are met:
14# 1. Redistributions of source code must retain the above copyright 14# 1. Redistributions of source code must retain the above copyright
15# notice, this list of conditions and the following disclaimer. 15# notice, this list of conditions and the following disclaimer.
@@ -343,59 +343,72 @@ sub adjust_gconf2_schemas() { @@ -343,59 +343,72 @@ sub adjust_gconf2_schemas() {
343 343
344sub adjust_libtool() { 344sub adjust_libtool() {
345 if (-f "$abs_wrksrc/ltconfig" || -f "$abs_wrksrc/ltmain.sh") { 345 if (-f "$abs_wrksrc/ltconfig" || -f "$abs_wrksrc/ltmain.sh") {
346 push(@build_vars, var("USE_LIBTOOL", "=", "yes")); 346 push(@build_vars, var("USE_LIBTOOL", "=", "yes"));
347 } 347 }
348 if (-d "$abs_wrksrc/libltdl") { 348 if (-d "$abs_wrksrc/libltdl") {
349 push(@includes, "../../devel/libltdl/convenience.mk"); 349 push(@includes, "../../devel/libltdl/convenience.mk");
350 } 350 }
351} 351}
352 352
353sub adjust_perl_module() { 353sub adjust_perl_module() {
354 354
355 if (-f "$abs_wrksrc/Build.PL") { 355 if (-f "$abs_wrksrc/Build.PL") {
 356 # Example packages:
 357 # devel/p5-Algorithm-CheckDigits
356 358
357 # It's a Module::Build module. Dependencies cannot yet be 359 # It's a Module::Build module. Dependencies cannot yet be
358 # extracted automatically. 360 # extracted automatically.
 361 #
 362 # TODO: Implement this similarly to the Makefile.PL mock below.
 363
359 push(@todos, "Look for the dependencies in Build.PL."); 364 push(@todos, "Look for the dependencies in Build.PL.");
360 365
361 push(@build_vars, var("PERL5_MODULE_TYPE", "=", "Module::Build")); 366 push(@build_vars, var("PERL5_MODULE_TYPE", "=", "Module::Build"));
362 367
363 } elsif (-f "$abs_wrksrc/Makefile.PL") { 368 } elsif (-f "$abs_wrksrc/Makefile.PL") {
 369 # Example packages:
 370 # devel/p5-Algorithm-Diff (no dependencies)
 371 # devel/p5-Carp-Assert-More (dependencies without version numbers)
 372 # www/p5-HTML-Quoted (dependency with version number)
364 373
365 # To avoid fix_up_makefile error for p5-HTML-Quoted, generate Makefile first. 374 # To avoid fix_up_makefile error for p5-HTML-Quoted, generate Makefile first.
366 system("cd '$abs_wrksrc' && perl -I. Makefile.PL < /dev/null") or do {}; 375 system("cd '$abs_wrksrc' && perl -I. Makefile.PL < /dev/null") or do {};
367 376
368 open(DEPS, "cd '$abs_wrksrc' && perl -I$libdir -I. Makefile.PL |") or die; 377 open(DEPS, "cd '$abs_wrksrc' && perl -I$libdir -I. Makefile.PL |") or die;
369 while (defined(my $dep = <DEPS>)) { 378 while (defined(my $dep = <DEPS>)) {
370 chomp($dep); 379 chomp($dep);
371 380
372 if ($dep =~ qr"^(\w+)\t(\S+)(>\S+|):(\.\./\.\./\S+)$") { 381 if ($dep =~ qr"^(\w+)\t(\S+)(>\S+|):(\.\./\.\./\S+)$") {
373 add_dependency($1, $2, $3, $4); 382 add_dependency($1, $2, $3, $4);
374 } 383 }
375 } 384 }
376 close(DEPS) or die; 385 close(DEPS) or die;
377 386
378 } else { 387 } else {
379 return; 388 return;
380 } 389 }
381 390
382 my $packlist = $distname =~ s/-[0-9].*//r =~ s/-/\//gr; 391 my $packlist = $distname =~ s/-[0-9].*//r =~ s/-/\//gr;
383 push(@build_vars, var("PERL5_PACKLIST", "=", "auto/$packlist/.packlist")); 392 push(@build_vars, var("PERL5_PACKLIST", "=", "auto/$packlist/.packlist"));
384 push(@includes, "../../lang/perl5/module.mk"); 393 push(@includes, "../../lang/perl5/module.mk");
385 $pkgname = "p5-\${DISTNAME}"; 394 $pkgname = "p5-\${DISTNAME}";
386 push(@categories, "perl5"); 395 push(@categories, "perl5");
 396 unlink("PLIST") or do {};
387} 397}
388 398
 399# Example packages:
 400#
 401# devel/py-ZopeComponent (dependencies, test dependencies)
389sub adjust_python_module() { 402sub adjust_python_module() {
390 403
391 return unless -f "$abs_wrksrc/setup.py"; 404 return unless -f "$abs_wrksrc/setup.py";
392 405
393 my %old_env = %ENV; 406 my %old_env = %ENV;
394 $ENV{"PYTHONDONTWRITEBYTECODE"} = "x"; 407 $ENV{"PYTHONDONTWRITEBYTECODE"} = "x";
395 $ENV{"PYTHONPATH"} = $libdir; 408 $ENV{"PYTHONPATH"} = $libdir;
396 409
397 my @dep_lines; 410 my @dep_lines;
398 open(DEPS, "cd '$abs_wrksrc' && $pythonbin setup.py build |") or die; 411 open(DEPS, "cd '$abs_wrksrc' && $pythonbin setup.py build |") or die;
399 %ENV = %old_env; 412 %ENV = %old_env;
400 while (defined(my $line = <DEPS>)) { 413 while (defined(my $line = <DEPS>)) {
401 chomp($line); 414 chomp($line);
@@ -616,47 +629,49 @@ sub generate_initial_package($) { @@ -616,47 +629,49 @@ sub generate_initial_package($) {
616 write_lines("DESCR", ()); 629 write_lines("DESCR", ());
617 run_editor("Makefile", 5); 630 run_editor("Makefile", 5);
618 631
619 make("distinfo"); 632 make("distinfo");
620 make("extract"); 633 make("extract");
621} 634}
622 635
623sub adjust_lines_python_module($$) { 636sub adjust_lines_python_module($$) {
624 my ($lines, $url) = @_; 637 my ($lines, $url) = @_;
625 638
626 my @initial_lines = generate_initial_package_Makefile_lines($url); 639 my @initial_lines = generate_initial_package_Makefile_lines($url);
627 my @current_lines = read_lines("Makefile"); 640 my @current_lines = read_lines("Makefile");
628 641
629 # don't risk to overwrite any changes made by the package developer. 
630 if (join('\n', @current_lines) ne join('\n', @initial_lines)) { 
631 splice(@$lines, -2, 0, "# TODO: Migrate MASTER_SITES to PYPI"); 
632 return; 
633 } 
634 
635 my %old; 642 my %old;
636 foreach my $line (@initial_lines) { 643 foreach my $line (@initial_lines) {
637 if ($line =~ qr"^(\w+)(\+?=)([ \t]+)([^#\\]*?)(\s*)(#.*|)$") { 644 if ($line =~ qr"^(\w+)(\+?=)([ \t]+)([^#\\]*?)(\s*)(#.*|)$") {
638 my ($varname, $op, $indent, $value, $space_after_value, $comment) = ($1, $2, $3, $4, $5, $6); 645 my ($varname, $op, $indent, $value, $space_after_value, $comment) = ($1, $2, $3, $4, $5, $6);
639 646
640 if ($op eq "=") { 647 if ($op eq "=") {
641 $old{$varname} = $value; 648 $old{$varname} = $value;
642 } 649 }
643 } 650 }
644 } 651 }
645 652
 653 return unless $old{"CATEGORIES"} =~ qr"python";
646 my $pkgbase = $old{"GITHUB_PROJECT"}; 654 my $pkgbase = $old{"GITHUB_PROJECT"};
 655 return unless defined($pkgbase);
647 my $pkgbase1 = substr($pkgbase, 0, 1); 656 my $pkgbase1 = substr($pkgbase, 0, 1);
648 my $pkgversion_norev = $old{"DISTNAME"} =~ s/^v//r; 657 my $pkgversion_norev = $old{"DISTNAME"} =~ s/^v//r;
649 658
 659 # don't risk to overwrite any changes made by the package developer.
 660 if (join('\n', @current_lines) ne join('\n', @initial_lines)) {
 661 splice(@$lines, -2, 0, "# TODO: Migrate MASTER_SITES to PYPI");
 662 return;
 663 }
 664
650 my @lines = @$lines; 665 my @lines = @$lines;
651 if (lines_remove(\@lines, "GITHUB_PROJECT") 666 if (lines_remove(\@lines, "GITHUB_PROJECT")
652 && lines_set(\@lines, "DISTNAME", "$pkgbase-$pkgversion_norev") 667 && lines_set(\@lines, "DISTNAME", "$pkgbase-$pkgversion_norev")
653 && lines_set(\@lines, "PKGNAME", "\${PYPKGPREFIX}-\${DISTNAME}") 668 && lines_set(\@lines, "PKGNAME", "\${PYPKGPREFIX}-\${DISTNAME}")
654 && lines_set(\@lines, "MASTER_SITES", "\${MASTER_SITE_PYPI:=$pkgbase1/$pkgbase/}") 669 && lines_set(\@lines, "MASTER_SITES", "\${MASTER_SITE_PYPI:=$pkgbase1/$pkgbase/}")
655 && lines_remove(\@lines, "DIST_SUBDIR") 670 && lines_remove(\@lines, "DIST_SUBDIR")
656 && (lines_remove_if(\@lines, "EXTRACT_SUFX", ".zip") || true)) { 671 && (lines_remove_if(\@lines, "EXTRACT_SUFX", ".zip") || true)) {
657 672
658 @$lines = @lines; 673 @$lines = @lines;
659 $regenerate_distinfo = true 674 $regenerate_distinfo = true
660 } 675 }
661} 676}
662 677