Sat Jan 1 15:29:14 2022 UTC ()
url2pkg: improve MASTER_SITES and HOMEPAGE for PyPI packages

https://mail-index.netbsd.org/tech-pkg/2021/12/31/msg025908.html

Bump version.


(rillig)
diff -r1.122 -r1.123 pkgsrc/pkgtools/url2pkg/Makefile
diff -r1.15 -r1.16 pkgsrc/pkgtools/url2pkg/files/url2pkg.8
diff -r1.34 -r1.35 pkgsrc/pkgtools/url2pkg/files/url2pkg.py
diff -r1.35 -r1.36 pkgsrc/pkgtools/url2pkg/files/url2pkg_test.py

cvs diff -r1.122 -r1.123 pkgsrc/pkgtools/url2pkg/Makefile (expand / switch to unified diff)

--- pkgsrc/pkgtools/url2pkg/Makefile 2022/01/01 14:04:11 1.122
+++ pkgsrc/pkgtools/url2pkg/Makefile 2022/01/01 15:29:14 1.123
@@ -1,16 +1,16 @@ @@ -1,16 +1,16 @@
1# $NetBSD: Makefile,v 1.122 2022/01/01 14:04:11 rillig Exp $ 1# $NetBSD: Makefile,v 1.123 2022/01/01 15:29:14 rillig Exp $
2 2
3PKGNAME= url2pkg-21.4.0 3PKGNAME= url2pkg-21.4.1
4CATEGORIES= pkgtools 4CATEGORIES= pkgtools
5 5
6MAINTAINER= rillig@NetBSD.org 6MAINTAINER= rillig@NetBSD.org
7HOMEPAGE= https://www.NetBSD.org/docs/pkgsrc/creating.html 7HOMEPAGE= https://www.NetBSD.org/docs/pkgsrc/creating.html
8COMMENT= Tool to automate initial steps in building a package 8COMMENT= Tool to automate initial steps in building a package
9LICENSE= 2-clause-bsd 9LICENSE= 2-clause-bsd
10 10
11WRKSRC= ${WRKDIR} 11WRKSRC= ${WRKDIR}
12NO_CHECKSUM= yes 12NO_CHECKSUM= yes
13NO_BUILD= yes 13NO_BUILD= yes
14USE_LANGUAGES= # none 14USE_LANGUAGES= # none
15USE_TOOLS+= perl:run 15USE_TOOLS+= perl:run
16AUTO_MKDIRS= yes 16AUTO_MKDIRS= yes

cvs diff -r1.15 -r1.16 pkgsrc/pkgtools/url2pkg/files/url2pkg.8 (expand / switch to unified diff)

--- pkgsrc/pkgtools/url2pkg/files/url2pkg.8 2022/01/01 14:04:11 1.15
+++ pkgsrc/pkgtools/url2pkg/files/url2pkg.8 2022/01/01 15:29:14 1.16
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1.\" $NetBSD: url2pkg.8,v 1.15 2022/01/01 14:04:11 rillig Exp $ 1.\" $NetBSD: url2pkg.8,v 1.16 2022/01/01 15:29:14 rillig Exp $
2.\" 2.\"
3.\" Copyright (c) 2001, 2019 The NetBSD Foundation, Inc. 3.\" Copyright (c) 2001, 2019 The NetBSD Foundation, Inc.
4.\" All rights reserved. 4.\" All rights reserved.
5.\" 5.\"
6.\" This code is derived from software contributed to The NetBSD Foundation 6.\" This code is derived from software contributed to The NetBSD Foundation
7.\" by Emmanuel Dreyfus and Roland Illig. 7.\" by Emmanuel Dreyfus and Roland Illig.
8.\" 8.\"
9.\" Redistribution and use in source and binary forms, with or without 9.\" Redistribution and use in source and binary forms, with or without
10.\" modification, are permitted provided that the following conditions 10.\" modification, are permitted provided that the following conditions
11.\" are met: 11.\" are met:
12.\" 1. Redistributions of source code must retain the above copyright 12.\" 1. Redistributions of source code must retain the above copyright
13.\" notice, this list of conditions and the following disclaimer. 13.\" notice, this list of conditions and the following disclaimer.
14.\" 2. Redistributions in binary form must reproduce the above copyright 14.\" 2. Redistributions in binary form must reproduce the above copyright
@@ -26,27 +26,27 @@ @@ -26,27 +26,27 @@
26.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28.\" POSSIBILITY OF SUCH DAMAGE. 28.\" POSSIBILITY OF SUCH DAMAGE.
29.\" 29.\"
30.Dd January 1, 2022 30.Dd January 1, 2022
31.Dt URL2PKG 8 31.Dt URL2PKG 8
32.Os 32.Os
33.Sh NAME 33.Sh NAME
34.Nm url2pkg 34.Nm url2pkg
35.Nd Automatic pkgsrc package generator 35.Nd Automatic pkgsrc package generator
36.Sh SYNOPSIS 36.Sh SYNOPSIS
37.Nm 37.Nm
38.Op Fl v|--verbose 38.Op Fl v|--verbose
39.Op Ar URL 39.Ar URL
40.Sh DESCRIPTION 40.Sh DESCRIPTION
41.Nm 41.Nm
42takes the 42takes the
43.Ar URL 43.Ar URL
44to a package's distfile that should be turned into a package. 44to a package's distfile that should be turned into a package.
45It fetches the distfile, extracts it, and by looking at the extracted files, 45It fetches the distfile, extracts it, and by looking at the extracted files,
46fills in some variables in the package 46fills in some variables in the package
47.Pa Makefile . 47.Pa Makefile .
48.Pp 48.Pp
49.Nm 49.Nm
50helps creating a package; 50helps creating a package;
51it is not intended to fly on autopilot, though. 51it is not intended to fly on autopilot, though.
52.Pp 52.Pp

cvs diff -r1.34 -r1.35 pkgsrc/pkgtools/url2pkg/files/url2pkg.py (expand / switch to unified diff)

--- pkgsrc/pkgtools/url2pkg/files/url2pkg.py 2022/01/01 14:04:11 1.34
+++ pkgsrc/pkgtools/url2pkg/files/url2pkg.py 2022/01/01 15:29:14 1.35
@@ -1,15 +1,15 @@ @@ -1,15 +1,15 @@
1#! @PYTHONBIN@ 1#! @PYTHONBIN@
2# $NetBSD: url2pkg.py,v 1.34 2022/01/01 14:04:11 rillig Exp $ 2# $NetBSD: url2pkg.py,v 1.35 2022/01/01 15:29:14 rillig Exp $
3 3
4# Copyright (c) 2019 The NetBSD Foundation, Inc. 4# Copyright (c) 2019 The NetBSD Foundation, Inc.
5# All rights reserved. 5# All rights reserved.
6# 6#
7# This code is derived from software contributed to The NetBSD Foundation 7# This code is derived from software contributed to The NetBSD Foundation
8# by Roland Illig. 8# by Roland Illig.
9# 9#
10# Redistribution and use in source and binary forms, with or without 10# Redistribution and use in source and binary forms, with or without
11# modification, are permitted provided that the following conditions 11# modification, are permitted provided that the following conditions
12# are met: 12# are met:
13# 1. Redistributions of source code must retain the above copyright 13# 1. Redistributions of source code must retain the above copyright
14# notice, this list of conditions and the following disclaimer. 14# notice, this list of conditions and the following disclaimer.
15# 2. Redistributions in binary form must reproduce the above copyright 15# 2. Redistributions in binary form must reproduce the above copyright
@@ -402,26 +402,48 @@ class Generator: @@ -402,26 +402,48 @@ class Generator:
402 ([^/?]+) # filename 402 ([^/?]+) # filename
403 (?:\?.*)? # query parameters 403 (?:\?.*)? # query parameters
404 $ 404 $
405 ''' 405 '''
406 m = re.search(pattern, self.url) 406 m = re.search(pattern, self.url)
407 if not m: 407 if not m:
408 return 408 return
409 409
410 project, subdir, filename = m.groups() 410 project, subdir, filename = m.groups()
411 self.master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/{subdir}}}' 411 self.master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/{subdir}}}'
412 self.homepage = f'https://{project}.sourceforge.net/' 412 self.homepage = f'https://{project}.sourceforge.net/'
413 self.distfile = filename 413 self.distfile = filename
414 414
 415 def adjust_site_PyPI(self):
 416 pattern = r'''(?x)
 417 ^
 418 https://files\.pythonhosted\.org/packages/
 419 ../../.{60}/ # hash
 420 ( # distfile
 421 (.*) # project
 422 -[0-9].* # version
 423 \.tar\.gz
 424 )
 425 $
 426 '''
 427 m = re.search(pattern, self.url)
 428 if not m:
 429 return
 430
 431 filename, project = m.groups()
 432 self.master_sites = f'${{MASTER_SITE_PYPI:={project[0]}/{project}/}}'
 433 self.homepage = f'https://pypi.org/project/{project}/'
 434 self.distfile = filename
 435
 436
415 def adjust_site_GitHub_archive(self): 437 def adjust_site_GitHub_archive(self):
416 pattern = r'''(?x) 438 pattern = r'''(?x)
417 ^ 439 ^
418 https://github\.com/ 440 https://github\.com/
419 (.+)/ # org 441 (.+)/ # org
420 (.+)/archive/ # proj 442 (.+)/archive/ # proj
421 ((?:.+/)? # tag 443 ((?:.+/)? # tag
422 (.+)) # distname 444 (.+)) # distname
423 (\.tar\.gz|\.zip) # ext 445 (\.tar\.gz|\.zip) # ext
424 $ 446 $
425 ''' 447 '''
426 m = re.search(pattern, self.url) 448 m = re.search(pattern, self.url)
427 if not m: 449 if not m:
@@ -535,26 +557,27 @@ class Generator: @@ -535,26 +557,27 @@ class Generator:
535 Var('#LICENSE', '=', '# TODO: (see mk/license.mk)'), 557 Var('#LICENSE', '=', '# TODO: (see mk/license.mk)'),
536 ) 558 )
537 559
538 lines.add('# url2pkg-marker (please do not remove this line.)') 560 lines.add('# url2pkg-marker (please do not remove this line.)')
539 lines.add('.include "../../mk/bsd.pkg.mk"') 561 lines.add('.include "../../mk/bsd.pkg.mk"')
540 562
541 return lines 563 return lines
542 564
543 def generate_Makefile(self) -> Lines: 565 def generate_Makefile(self) -> Lines:
544 self.adjust_site_SourceForge() 566 self.adjust_site_SourceForge()
545 self.adjust_site_GitHub_archive() 567 self.adjust_site_GitHub_archive()
546 self.adjust_site_GitHub_release() 568 self.adjust_site_GitHub_release()
547 self.foreach_site_from_sites_mk(self.adjust_site_from_sites_mk) 569 self.foreach_site_from_sites_mk(self.adjust_site_from_sites_mk)
 570 self.adjust_site_PyPI()
548 self.adjust_site_other() 571 self.adjust_site_other()
549 self.adjust_everything_else() 572 self.adjust_everything_else()
550 return self.generate_lines() 573 return self.generate_lines()
551 574
552 def generate_package(self, g: Globals) -> Lines: 575 def generate_package(self, g: Globals) -> Lines:
553 pkgdir = g.pkgdir 576 pkgdir = g.pkgdir
554 makefile = pkgdir / 'Makefile' 577 makefile = pkgdir / 'Makefile'
555 plist = pkgdir / 'PLIST' 578 plist = pkgdir / 'PLIST'
556 579
557 initial_lines = self.generate_Makefile() 580 initial_lines = self.generate_Makefile()
558 581
559 try: 582 try:
560 makefile.replace(f'{makefile}.url2pkg~') 583 makefile.replace(f'{makefile}.url2pkg~')
@@ -1171,27 +1194,27 @@ class Adjuster: @@ -1171,27 +1194,27 @@ class Adjuster:
1171 self.adjust_pkg_config() 1194 self.adjust_pkg_config()
1172 self.adjust_po() 1195 self.adjust_po()
1173 self.adjust_use_languages() 1196 self.adjust_use_languages()
1174 1197
1175 self.generate_lines().write_to(self.g.pkgdir / 'Makefile') 1198 self.generate_lines().write_to(self.g.pkgdir / 'Makefile')
1176 descr = (self.g.pkgdir / 'DESCR') 1199 descr = (self.g.pkgdir / 'DESCR')
1177 descr.is_file() or Lines(*self.descr_lines).write_to(descr) 1200 descr.is_file() or Lines(*self.descr_lines).write_to(descr)
1178 1201
1179 if self.regenerate_distinfo: 1202 if self.regenerate_distinfo:
1180 self.g.bmake('distinfo') 1203 self.g.bmake('distinfo')
1181 1204
1182 1205
1183def usage(): 1206def usage():
1184 sys.exit(f'usage: {sys.argv[0]} [-v|--verbose] [URL]') 1207 sys.exit(f'usage: {sys.argv[0]} [-v|--verbose] URL')
1185 1208
1186 1209
1187def main(argv: List[str], g: Globals): 1210def main(argv: List[str], g: Globals):
1188 if not os.path.isfile('../../mk/bsd.pkg.mk'): 1211 if not os.path.isfile('../../mk/bsd.pkg.mk'):
1189 sys.exit(f'{argv[0]}: must be run from a package directory ' 1212 sys.exit(f'{argv[0]}: must be run from a package directory '
1190 f'(.../pkgsrc/category/package)') 1213 f'(.../pkgsrc/category/package)')
1191 1214
1192 try: 1215 try:
1193 opts, args = getopt.getopt(argv[1:], 'v', ['verbose']) 1216 opts, args = getopt.getopt(argv[1:], 'v', ['verbose'])
1194 for (opt, _) in opts: 1217 for (opt, _) in opts:
1195 if opt in ('-v', '--verbose'): 1218 if opt in ('-v', '--verbose'):
1196 g.verbose = True 1219 g.verbose = True
1197 except getopt.GetoptError: 1220 except getopt.GetoptError:

cvs diff -r1.35 -r1.36 pkgsrc/pkgtools/url2pkg/files/url2pkg_test.py (expand / switch to unified diff)

--- pkgsrc/pkgtools/url2pkg/files/url2pkg_test.py 2022/01/01 15:04:58 1.35
+++ pkgsrc/pkgtools/url2pkg/files/url2pkg_test.py 2022/01/01 15:29:14 1.36
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1# $NetBSD: url2pkg_test.py,v 1.35 2022/01/01 15:04:58 rillig Exp $ 1# $NetBSD: url2pkg_test.py,v 1.36 2022/01/01 15:29:14 rillig Exp $
2 2
3import pytest 3import pytest
4from url2pkg import * 4from url2pkg import *
5from textwrap import dedent 5from textwrap import dedent
6 6
7mkcvsid = '# $''NetBSD$' 7mkcvsid = '# $''NetBSD$'
8g: Globals 8g: Globals
9prev_dir = Path.cwd() 9prev_dir = Path.cwd()
10 10
11 11
12def setup_function(_): 12def setup_function(_):
13 global g 13 global g
14 14
@@ -603,34 +603,30 @@ def test_Generator_adjust_site_from_site @@ -603,34 +603,30 @@ def test_Generator_adjust_site_from_site
603def test_Generator_adjust_site_from_sites_mk__PyPI(): 603def test_Generator_adjust_site_from_sites_mk__PyPI():
604 url = ('https://files.pythonhosted.org/' 604 url = ('https://files.pythonhosted.org/'
605 + 'packages/da/8b/218264f5ce91df1ad27ce8021d51b747ef287627338fe05d170565358546/' 605 + 'packages/da/8b/218264f5ce91df1ad27ce8021d51b747ef287627338fe05d170565358546/'
606 + 'apprise-0.9.6.tar.gz') 606 + 'apprise-0.9.6.tar.gz')
607 generator = Generator(url) 607 generator = Generator(url)
608 608
609 lines = generator.generate_Makefile() 609 lines = generator.generate_Makefile()
610 610
611 assert detab(lines) == [ 611 assert detab(lines) == [
612 mkcvsid, 612 mkcvsid,
613 '', 613 '',
614 'DISTNAME= apprise-0.9.6', 614 'DISTNAME= apprise-0.9.6',
615 'CATEGORIES= pkgtools', 615 'CATEGORIES= pkgtools',
616 # TODO: ${MASTER_SITE_PYPI:=a/apprise/} 616 'MASTER_SITES= ${MASTER_SITE_PYPI:=a/apprise/}',
617 'MASTER_SITES= https://files.pythonhosted.org/packages/da/8b/' 
618 + '218264f5ce91df1ad27ce8021d51b747ef287627338fe05d170565358546/', 
619 '', 617 '',
620 'MAINTAINER= INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org', 618 'MAINTAINER= INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org',
621 # TODO: https://pypi.org/project/apprise/ 619 'HOMEPAGE= https://pypi.org/project/apprise/',
622 'HOMEPAGE= https://files.pythonhosted.org/packages/da/8b/' 
623 + '218264f5ce91df1ad27ce8021d51b747ef287627338fe05d170565358546/', 
624 'COMMENT= TODO: Short description of the package', 620 'COMMENT= TODO: Short description of the package',
625 '#LICENSE= # TODO: (see mk/license.mk)', 621 '#LICENSE= # TODO: (see mk/license.mk)',
626 '', 622 '',
627 '# url2pkg-marker (please do not remove this line.)', 623 '# url2pkg-marker (please do not remove this line.)',
628 '.include "../../mk/bsd.pkg.mk"', 624 '.include "../../mk/bsd.pkg.mk"',
629 ] 625 ]
630 626
631 627
632def test_Generator_adjust_site_from_sites_mk__R(tmp_path: Path): 628def test_Generator_adjust_site_from_sites_mk__R(tmp_path: Path):
633 g.pkgdir = tmp_path 629 g.pkgdir = tmp_path
634 url = 'http://cran.r-project.org/src/contrib/forecast_8.7.tar.gz' 630 url = 'http://cran.r-project.org/src/contrib/forecast_8.7.tar.gz'
635 generator = Generator(url) 631 generator = Generator(url)
636 632