 @@ -1,16 +1,16 @@
-# $NetBSD: Makefile,v 1.106 2019/10/06 12:55:30 rillig Exp $
+# $NetBSD: Makefile,v 1.107 2019/10/27 19:19:54 rillig Exp $
-PKGNAME=	url2pkg-19.3.3
+PKGNAME=	url2pkg-19.3.4
 CATEGORIES=	pkgtools
 MAINTAINER=	rillig@NetBSD.org
 HOMEPAGE=	http://www.NetBSD.org/docs/pkgsrc/creating.html
 COMMENT=	Tool to automate initial steps in building a package
 LICENSE=	2-clause-bsd
 WRKSRC=		${WRKDIR}
 NO_CHECKSUM=	yes
 NO_BUILD=	yes
 USE_LANGUAGES=	# none
 USE_TOOLS+=	perl:run
 AUTO_MKDIRS=	yes
 @@ -31,16 +31,17 @@ SUBST_STAGE.up=		do-configure
 SUBST_MESSAGE.up=	Replacing variable placeholders
 SUBST_FILES.up=		MakeMaker.pm url2pkg.py
 SUBST_VARS.up=		MAKE PERL5 PKGSRCDIR PYTHONBIN
 SUBST_SED.up=		-e 's,@LIBDIR@,${PREFIX}/lib/url2pkg,g'
 do-install:
 	${INSTALL_SCRIPT} ${WRKSRC}/url2pkg.py ${DESTDIR}${PREFIX}/bin/url2pkg
 	${INSTALL_MAN} ${FILESDIR}/url2pkg.8 ${DESTDIR}${PREFIX}/${PKGMANDIR}/man8
 	${INSTALL_DATA} ${WRKSRC}/Build.pm ${DESTDIR}${PREFIX}/lib/url2pkg/Module/
 	${INSTALL_DATA} ${WRKSRC}/MakeMaker.pm ${DESTDIR}${PREFIX}/lib/url2pkg/ExtUtils/
 	${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/
 	${INSTALL_DATA} /dev/null ${DESTDIR}${PREFIX}/lib/url2pkg/distutils/__init__.py
 	${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/distutils/core.py
 	${INSTALL_DATA} ${WRKSRC}/setuptools.py ${DESTDIR}${PREFIX}/lib/url2pkg/distutils/extension.py
 .include "../../lang/python/application.mk"
 .include "../../mk/bsd.pkg.mk"

 @@ -1,8 +1,9 @@
-@comment $NetBSD: PLIST,v 1.8 2019/10/06 12:55:30 rillig Exp $
+@comment $NetBSD: PLIST,v 1.9 2019/10/27 19:19:54 rillig Exp $
 bin/url2pkg
 lib/url2pkg/ExtUtils/MakeMaker.pm
 lib/url2pkg/Module/Build.pm
 lib/url2pkg/distutils/__init__.py
 lib/url2pkg/distutils/core.py
 lib/url2pkg/distutils/extension.py
 lib/url2pkg/setuptools.py
 man/man8/url2pkg.8

 @@ -46,44 +46,48 @@ sub url2pkg_write_dependencies($$$) {
 	my $deps = $self->{$key};
 	foreach my $item (keys %$deps) {
 		my $pkgbase = "p5-$item" =~ s/::/-/gr;
 		printf("%s\t%s>=%s\n", $varname, $pkgbase, $deps->{$item});
+	}
+}
 sub url2pkg_write_var($$$) {
 	my ($self, $varname, $value) = @_;
 	return unless defined($value) && $value ne "";
 	printf("var\t%s\t%s\n", $varname, $value);
+}
 sub url2pkg_write_cmd($$) {
 	my ($self, $cmd, $arg) = @_;
 	printf("cmd\t%s\t%s\n", $cmd, $arg);
+}
 sub VERSION($$) {
 	my ($class, $min_version) = @_;
 	return $min_version <= $VERSION;
+}
 sub new(%) {
 	my ($class, %args) = @_;
 	my $self = \%args;
 	bless($self, $class);
 	return $self;
+}
 sub create_build_script($) {
 	my ($self) = @_;
 	$self->url2pkg_write_dependencies("BUILD_DEPENDS", "configure_requires");
 	$self->url2pkg_write_dependencies("BUILD_DEPENDS", "build_requires");
 	$self->url2pkg_write_dependencies("DEPENDS", "requires");
 	$self->url2pkg_write_dependencies("TEST_DEPENDS", "test_requires");
 	$self->url2pkg_write_dependencies("#RECOMMENDS", "recommends");
 	$self->url2pkg_write_var("COMMENT", $self->{"dist_abstract"});
 	my $license = $self->{"license"} || "";
-	if ($license eq "perl") {
+	if ($license ne "") {
-		$self->url2pkg_write_var("LICENSE", "\${PERL5_LICENSE}");
+		$self->url2pkg_write_cmd("license", $license);
-	} elsif ($license ne "") {
+		$self->url2pkg_write_var("license_default", "# TODO: $license (from Build.PL)")
 		$self->url2pkg_write_var("#LICENSE", "# TODO: $license (from Build.PL)")
+	}
+}
 ;

 @@ -89,46 +89,50 @@ sub url2pkg_write_dependency($$$) {
 	# If the package does not exist but the Perl module can be loaded, assume
 	# that it is a built-in module and no dependency declaration is needed.
 	return if eval("use $dep $ver; 1;");
 	printf("%s\t%s>=%s\n", $type, $pkgbase, $ver);
+}
 sub url2pkg_write_var($$) {
 	my ($varname, $value) = @_;
 	return unless defined($value) && $value ne "";
 	printf("var\t%s\t%s\n", $varname, $value);
+}
 sub url2pkg_write_cmd($$) {
 	my ($cmd, $arg) = @_;
 	printf("cmd\t%s\t%s\n", $cmd, $arg);
+}
 sub url2pkg_write_depends($$) {
 	my ($type, $deps) = @_;
-	return unless $deps;
+	return unless defined $deps;
 	foreach my $dep (sort(keys(%$deps))) {
 		url2pkg_write_dependency($type, $dep, $deps->{$dep});
+	}
+}
 sub WriteMakefile(%) {
 	my (%options) = @_;
 	url2pkg_write_depends("DEPENDS", $options{"PREREQ_PM"});
 	url2pkg_write_depends("TEST_DEPENDS", $options{"TEST_DEPENDS"});
 	my $license = $options{"LICENSE"} || "";
-	if ($license eq "perl") {
+	if ($license ne "") {
-		url2pkg_write_var("LICENSE", "\${PERL5_LICENSE}");
+		url2pkg_write_cmd("license", $license);
-	} elsif ($license ne "") {
+		url2pkg_write_cmd("license_default", "# TODO: $license (from Makefile.PL)");
 		url2pkg_write_var("#LICENSE", "# TODO: $license (from Makefile.PL)")
+	}
+}
 sub prompt(@) {
 	my ($message, $default) = @_;
 	return $default || "";
+}
 sub neatvalue {
 	return;
+}

 @@ -1,14 +1,14 @@
-# $NetBSD: setuptools.py,v 1.6 2019/10/27 13:15:04 rillig Exp $
+# $NetBSD: setuptools.py,v 1.7 2019/10/27 19:19:55 rillig Exp $
 # This is a drop-in replacement for the setuptools Python module. Instead
 # of actually searching for the dependencies, it extracts the dependency
 # information and includes it in the generated pkgsrc package Makefile.
 def url2pkg_print_depends(varname, depends):
     for dep in depends:
         print('%s\t%s%s' % (varname, dep.replace(' ', ''), '' if '>' in dep else '>=0'))
 def url2pkg_print_var(varname, value):
     if value != '':
 @@ -34,13 +34,24 @@ def setup(**kwargs):
     url2pkg_print_depends('TEST_DEPENDS', kwargs.get('tests_require', []))
     url2pkg_print_depends('BUILD_DEPENDS', kwargs.get('extras_require', {}).get('dev', []))
     url2pkg_print_depends('BUILD_DEPENDS', kwargs.get('setup_requires', []))
     url2pkg_print_var('COMMENT', kwargs.get('description', '').rstrip('.'))
     url2pkg_print_var('HOMEPAGE', kwargs.get('url', ''))
     url2pkg_print_license(kwargs.get('license', ''))
     # TODO: python_requires (see devel/py-futures)
     #       example: '>=2.6, <3, >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*')
 def find_packages(where='.', exclude=(), include=('*',)):
     return []
 # for distutils.core; see devel/py-pysha3-1.0.2 for an example
 class Command:
     pass
 # for distutils.extension; see devel/py-pysha3-1.0.2 for an example
 class Extension:
     def __init__(self, *args, **kwargs):
         pass

 @@ -1,15 +1,15 @@
 #! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.23 2019/10/27 13:15:04 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.24 2019/10/27 19:19:55 rillig Exp $
 # Copyright (c) 2019 The NetBSD Foundation, Inc.
 # All rights reserved.
+#
 # This code is derived from software contributed to The NetBSD Foundation
 # by Roland Illig.
+#
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 @@ -112,45 +112,47 @@ class Globals:
         subprocess.check_call([self.make, *args], cwd=self.pkgdir)
     def show_var(self, varname: str) -> str:
         output = subprocess.check_output((self.make, 'show-var', 'VARNAME=' + varname))
         return output.decode('utf-8').strip()
     def pkgsrc_license(self, license_name: str) -> str:
         known_licenses = (
             ('',
              'Apache Software License',  # too unspecific; needs a version number
              'BSD'),  # too unspecific, may be 2-clause, 3-clause, 4-clause
             ('${PERL5_LICENSE}', 'perl'),
             ('apache-2.0', 'Apache 2', 'Apache 2.0'),
             ('artistic-2.0', 'artistic_2'),
             ('gnu-gpl-v3', 'GNU Lesser General Public License (LGPL), Version 3'),
             ('gnu-lgpl-v2', 'LGPL'),
             ('mit', 'MIT', 'MIT License'),
             ('python-software-foundation',
              'PSF', 'PSF license', 'Python Software Foundation License'),
             ('zpl-2.1', 'ZPL 2.1'),
+        )
         for known_license in known_licenses:
             if license_name in known_license:
                 return known_license[0]
         if (self.pkgsrcdir / 'licenses' / license_name).is_file():
             return license_name
         return ''
 class Lines:
     """
-    A list of Makefile lines with high-level methods for manipulating
+    A list of lines (typically from a Makefile, but other file types work as
-    variable assignments.
+    well) with high-level methods for manipulating variable assignments.
     """
     lines: List[str]
     def __init__(self, *lines: str) -> None:
         self.lines = []
         for line in lines:
             self.add(line)
     @classmethod
     def read_from(cls, src: Union[Path, Any]) -> 'Lines':
         return Lines(*src.read_text().splitlines())
     def write_to(self, dst: Union[Path, Any]):
 @@ -473,38 +475,43 @@ class Generator:
     def generate_Makefile(self) -> Lines:
         self.adjust_site_SourceForge()
         self.adjust_site_GitHub_archive()
         self.adjust_site_GitHub_release()
         self.foreach_site_from_sites_mk(self.adjust_site_from_sites_mk)
         self.adjust_site_other()
         self.adjust_everything_else()
         return self.generate_lines()
     def generate_package(self, g: Globals) -> Lines:
         pkgdir = g.pkgdir
         makefile = pkgdir / 'Makefile'
         descr = pkgdir / 'DESCR'
         plist = pkgdir / 'PLIST'
         initial_lines = self.generate_Makefile()
         try:
             makefile.replace(f'{makefile}.url2pkg~')
         except OSError:
             pass
         initial_lines.write_to(makefile)
         plist.is_file() or Lines('@comment $''NetBSD$').write_to(plist)
-        descr.is_file() or Lines().write_to(descr)
+        plist_lines = [
             f'@comment $''NetBSD$',
             f'@comment TODO: to fill this file with the file listing:',
             f'@comment TODO: 1. run "{g.make} package"',
             f'@comment TODO: 2. run "{g.make} print-PLIST"'
+        ]
         plist.is_file() or Lines(*plist_lines).write_to(plist)
         subprocess.check_call([g.editor, makefile])
         g.bmake('clean', 'distinfo', 'extract')
         return initial_lines
 class Adjuster:
     """
     After the distfile has been downloaded and extracted, the
     adjust_* methods of this class inspect the extracted files
     and adjust the variable definitions in the package Makefile.
 @@ -560,47 +567,50 @@ class Adjuster:
     # these are inserted below the second paragraph in the Makefile.
     todos: List[str]
     # the package name is based on DISTNAME and modified by
     # pkgname_prefix and pkgname_transform.
     pkgname_prefix: str  # example: ${PYPKGPREFIX}-
     pkgname_transform: str  # example: :S,-v,-,
     # all lines of the package Makefile, for direct modification.
     makefile_lines: Lines
     regenerate_distinfo: bool
     descr_lines: List[str]
     def __init__(self, g: Globals, url: str, initial_lines: Lines):
         self.g = g
         self.url = url
         self.initial_lines = initial_lines
         self.abs_wrkdir = Path('')
         self.abs_wrksrc = Path('')
         self.wrksrc_files = []
         self.wrksrc_dirs = []
         self.categories = []
         self.depends = []
         self.build_depends = []
         self.test_depends = []
         self.bl3_lines = []
         self.includes = []
         self.build_vars = []
         self.extra_vars = []
         self.todos = []
         self.pkgname_prefix = ''
         self.pkgname_transform = ''
         self.makefile_lines = Lines()
         self.regenerate_distinfo = False
         self.descr_lines = []
     def add_dependency(self, kind: str, pkgbase: str, constraint: str, dep_dir: str) -> None:
         """ add_dependency('DEPENDS', 'package', '>=1', '../../category/package') """
         self.g.debug('add_dependency: {0} {1} {2} {3}', kind, pkgbase, constraint, dep_dir)
         def bl3_identifier():
             try:
                 with open(dep_dir + '/buildlink3.mk') as f:
                     for line in f:
                         m = re.search(r'^BUILDLINK_TREE\+=\s*(\S+)$', line)
                         if m:
                             return m[1]
 @@ -688,49 +698,72 @@ class Adjuster:
         if not self.makefile_lines.set(varname, value):
             self.extra_vars.append(Var(varname, '=', value))
     def set_license(self, license_name: str, license_default: str):
         pkgsrc_license_name = self.g.pkgsrc_license(license_name)
         if pkgsrc_license_name != '':
             self.set_or_add('LICENSE', pkgsrc_license_name)
         elif license_default != '':
             self.set_or_add('#LICENSE', license_default)
     def wrksrc_open(self, relative_pathname: str):
         return (self.abs_wrksrc / relative_pathname).open()
     def wrksrc_head(self, relative_pathname: str, n: int):
         try:
             with (self.abs_wrksrc / relative_pathname).open(encoding="UTF-8") as f:
                 return f.read().splitlines()[:n]
         except IOError:
             return []
     def wrksrc_find(self, what: Union[str, Callable[[str], bool]]) -> List[str]:
         def search(f):
             return re.search(what, f) if type(what) == str else what(f)
         return list(sorted(filter(search, self.wrksrc_files)))
     def wrksrc_grep(self, filename: str, pattern: str) -> List[Union[str, List[str]]]:
         with self.wrksrc_open(filename) as f:
             matches = []
             for line in f:
                 line = line.rstrip('\n')
                 m = re.search(pattern, line)
                 if m:
                     groups = list(m.groups())
                     matches.append(groups if groups else line)
             return matches
     def wrksrc_isdir(self, relative_pathname: str) -> bool:
         return (self.abs_wrksrc / relative_pathname).is_dir()
     def wrksrc_isfile(self, relative_pathname: str) -> bool:
         return (self.abs_wrksrc / relative_pathname).is_file()
     def adjust_descr(self):
         for filename in ('README', 'README.txt', 'README.md'):
             lines = self.wrksrc_head(filename, 21)
             if len(lines) == 21:
                 lines[-1] = '...'
             if lines:
                 self.descr_lines = [
                     f'TODO: Adjust the following lines from {filename}',
                     '',
                     *lines]
                 return
         self.descr_lines = [
             'TODO: Fill in a short description of the package.',
             'TODO: It should be between 3 and 20 lines.']
     def adjust_configure(self):
         if not self.wrksrc_isfile('configure'):
             return
         configures = self.wrksrc_find(r'(^|/)configure$')
         if configures:
             gnu = any(self.wrksrc_grep(configure, r'\b(Free Software Foundation|autoconf)\b')
                       for configure in configures)
             varname = 'GNU_CONFIGURE' if gnu else 'HAS_CONFIGURE'
             self.build_vars.append(Var(varname, '=', 'yes'))
     def adjust_cmake(self):
         if self.wrksrc_isfile('CMakeLists.txt'):
 @@ -755,27 +788,27 @@ class Adjuster:
         if self.wrksrc_isdir('libltdl'):
             self.includes.append('../../devel/libltdl/convenience.mk')
     def adjust_perl_module_Build_PL(self):
         # Example packages:
         # devel/p5-Algorithm-CheckDigits
         cmd = f'{self.g.perl5} -I{self.g.libdir} -I. Build.PL'
         self.read_dependencies(cmd, {}, self.abs_wrksrc, '')
         self.build_vars.append(Var('PERL5_MODULE_TYPE', '=', 'Module::Build'))
     def adjust_perl_module_Makefile_PL(self):
         # Example packages:
-        # devel/p5-Algorithm-Diff (no dependencies)
+        # devel/p5-Algorithm-Diff (no dependencies, no license)
         # devel/p5-Carp-Assert-More (dependencies without version numbers)
         # www/p5-HTML-Quoted (dependency with version number)
         # To avoid fix_up_makefile error for p5-HTML-Quoted, generate Makefile first.
         cmd1 = f'{self.g.perl5} -I. Makefile.PL </dev/null 1>&0 2>&0'
         subprocess.call(cmd1, shell=True, cwd=self.abs_wrksrc)
         cmd2 = f'{self.g.perl5} -I{self.g.libdir} -I. Makefile.PL'
         self.read_dependencies(cmd2, {}, self.abs_wrksrc, '')
     def adjust_perl_module_homepage(self):
         if '${MASTER_SITE_PERL_CPAN:' not in self.makefile_lines.get('MASTER_SITES'):
             return
 @@ -980,39 +1013,42 @@ class Adjuster:
         def scan(basedir: Union[Path, Any], only: Callable[[Path], bool]) -> List[str]:
             relevant = (f for f in basedir.rglob('*') if only(f))
             relative = (str(f.relative_to(basedir)) for f in relevant)
             return list(sorted((f for f in relative if not f.startswith('.'))))
         self.g.debug('Adjusting the Makefile')
         self.makefile_lines = Lines.read_from(self.g.pkgdir / 'Makefile')
         self.abs_wrkdir = Path(self.g.show_var('WRKDIR'))
         self.determine_wrksrc()
         self.wrksrc_dirs = scan(self.abs_wrksrc, Path.is_dir)
         self.wrksrc_files = scan(self.abs_wrksrc, Path.is_file)
         self.adjust_descr()
         self.adjust_configure()
         self.adjust_cmake()
         self.adjust_meson()
         self.adjust_gconf2_schemas()
         self.adjust_libtool()
         self.adjust_perl_module()
         self.adjust_python_module()
         self.adjust_cargo()
         self.adjust_pkg_config()
         self.adjust_po()
         self.adjust_use_languages()
         self.generate_lines().write_to(self.g.pkgdir / 'Makefile')
         descr = (self.g.pkgdir / 'DESCR')
         descr.is_file() or Lines(*self.descr_lines).write_to(descr)
         if self.regenerate_distinfo:
             self.g.bmake('distinfo')
 def main(argv: List[str], g: Globals):
     if not os.path.isfile('../../mk/bsd.pkg.mk'):
         sys.exit(f'{argv[0]}: must be run from a package directory (.../pkgsrc/category/package)')
     try:
         opts, args = getopt.getopt(argv[1:], 'v', ['verbose'])
         for (opt, _) in opts:
             if opt in ('-v', '--verbose'):

 @@ -1,14 +1,14 @@
-# $NetBSD: url2pkg_test.py,v 1.22 2019/10/27 13:15:04 rillig Exp $
+# $NetBSD: url2pkg_test.py,v 1.23 2019/10/27 19:19:55 rillig Exp $
 import pytest
 from url2pkg import *
 mkcvsid = '# $''NetBSD$'
 g: Globals
 prev_dir = Path.cwd()
 def setup_function(_):
     global g
     g = Globals()
 @@ -89,27 +89,27 @@ def test_Global_bmake():
     g.bmake('hello', 'world')
     assert g.err.written() == [
         'url2pkg: running bmake (\'hello\', \'world\') in \'.\'',
+    ]
 def test_Global_pkgsrc_license():
     assert g.pkgsrc_license('BSD') == ''
     assert g.pkgsrc_license('apache-2.0') == 'apache-2.0'
     assert g.pkgsrc_license('Apache 2') == 'apache-2.0'
     # Not explicitly in the list, looked up from PKGSRCDIR.
-    assert g.pkgsrc_license('artistic-2.0') == 'artistic-2.0'
+    assert g.pkgsrc_license('skype21-license') == 'skype21-license'
     # Neither in the list nor in PKGSRCDIR/licenses.
     assert g.pkgsrc_license('unknown') == ''
 def test_Lines__write_and_read(tmp_path: Path):
     example = tmp_path / 'example'
     lines = Lines('1', '2', '3')
     lines.write_to(example)
     assert example.read_text() == '1\n2\n3\n'
 @@ -637,32 +637,37 @@ def test_Generator_adjust_everything_els
         '# url2pkg-marker (please do not remove this line.)',
         '.include "../../mk/bsd.pkg.mk"',
+    ]
 def test_Generator_generate_package(tmp_path: Path):
     url = 'https://ftp.gnu.org/pub/gnu/cflow/cflow-1.6.tar.gz'
     g.editor = 'true'  # the shell command
     g.make = 'true'  # the shell command
     g.pkgdir = tmp_path
     Generator(url).generate_package(g)
-    assert (tmp_path / 'DESCR').read_text() == ''
+    assert not (tmp_path / 'DESCR').is_file()  # is created later
     assert len((tmp_path / 'Makefile').read_text().splitlines()) == 13
-    assert (tmp_path / 'PLIST').read_text() == '@comment $''NetBSD$\n'
+    assert (tmp_path / 'PLIST').read_text().splitlines() == [
         '@comment $''NetBSD$',
         '@comment TODO: to fill this file with the file listing:',
         '@comment TODO: 1. run "true package"',
         '@comment TODO: 2. run "true print-PLIST"',
+    ]
     # Since bmake is only fake in this test, the distinfo file is not created.
-    expected_files = ['DESCR', 'Makefile', 'PLIST']
+    expected_files = ['Makefile', 'PLIST']
     assert sorted([f.name for f in tmp_path.glob("*")]) == expected_files
 def test_Adjuster_read_dependencies():
     child_process_output = [
         'DEPENDS\tpackage>=112.0:../../pkgtools/pkglint',
         'DEPENDS\tpackage>=120.0:../../pkgtools/x11-links',
         'BUILD_DEPENDS\turl2pkg>=1.0',
         'BUILD_DEPENDS\tdoes-not-exist>=1.0',
         'TEST_DEPENDS\tpkglint',
         'A line that is not a dependency at all',
         '',
         'var\tHOMEPAGE\thttps://homepage.example.org/',