url2pkg: improve MASTER_SITES and HOMEPAGE for PyPI packages https://mail-index.netbsd.org/tech-pkg/2021/12/31/msg025908.html Bump version.diff -r1.122 -r1.123 pkgsrc/pkgtools/url2pkg/Makefile
(rillig)
@@ -1,16 +1,16 @@ | @@ -1,16 +1,16 @@ | |||
1 | # $NetBSD: Makefile,v 1.122 2022/01/01 14:04:11 rillig Exp $ | 1 | # $NetBSD: Makefile,v 1.123 2022/01/01 15:29:14 rillig Exp $ | |
2 | 2 | |||
3 | PKGNAME= url2pkg-21.4.0 | 3 | PKGNAME= url2pkg-21.4.1 | |
4 | CATEGORIES= pkgtools | 4 | CATEGORIES= pkgtools | |
5 | 5 | |||
6 | MAINTAINER= rillig@NetBSD.org | 6 | MAINTAINER= rillig@NetBSD.org | |
7 | HOMEPAGE= https://www.NetBSD.org/docs/pkgsrc/creating.html | 7 | HOMEPAGE= https://www.NetBSD.org/docs/pkgsrc/creating.html | |
8 | COMMENT= Tool to automate initial steps in building a package | 8 | COMMENT= Tool to automate initial steps in building a package | |
9 | LICENSE= 2-clause-bsd | 9 | LICENSE= 2-clause-bsd | |
10 | 10 | |||
11 | WRKSRC= ${WRKDIR} | 11 | WRKSRC= ${WRKDIR} | |
12 | NO_CHECKSUM= yes | 12 | NO_CHECKSUM= yes | |
13 | NO_BUILD= yes | 13 | NO_BUILD= yes | |
14 | USE_LANGUAGES= # none | 14 | USE_LANGUAGES= # none | |
15 | USE_TOOLS+= perl:run | 15 | USE_TOOLS+= perl:run | |
16 | AUTO_MKDIRS= yes | 16 | AUTO_MKDIRS= yes |
@@ -1,14 +1,14 @@ | @@ -1,14 +1,14 @@ | |||
1 | .\" $NetBSD: url2pkg.8,v 1.15 2022/01/01 14:04:11 rillig Exp $ | 1 | .\" $NetBSD: url2pkg.8,v 1.16 2022/01/01 15:29:14 rillig Exp $ | |
2 | .\" | 2 | .\" | |
3 | .\" Copyright (c) 2001, 2019 The NetBSD Foundation, Inc. | 3 | .\" Copyright (c) 2001, 2019 The NetBSD Foundation, Inc. | |
4 | .\" All rights reserved. | 4 | .\" All rights reserved. | |
5 | .\" | 5 | .\" | |
6 | .\" This code is derived from software contributed to The NetBSD Foundation | 6 | .\" This code is derived from software contributed to The NetBSD Foundation | |
7 | .\" by Emmanuel Dreyfus and Roland Illig. | 7 | .\" by Emmanuel Dreyfus and Roland Illig. | |
8 | .\" | 8 | .\" | |
9 | .\" Redistribution and use in source and binary forms, with or without | 9 | .\" Redistribution and use in source and binary forms, with or without | |
10 | .\" modification, are permitted provided that the following conditions | 10 | .\" modification, are permitted provided that the following conditions | |
11 | .\" are met: | 11 | .\" are met: | |
12 | .\" 1. Redistributions of source code must retain the above copyright | 12 | .\" 1. Redistributions of source code must retain the above copyright | |
13 | .\" notice, this list of conditions and the following disclaimer. | 13 | .\" notice, this list of conditions and the following disclaimer. | |
14 | .\" 2. Redistributions in binary form must reproduce the above copyright | 14 | .\" 2. Redistributions in binary form must reproduce the above copyright | |
@@ -26,27 +26,27 @@ | @@ -26,27 +26,27 @@ | |||
26 | .\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | 26 | .\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
27 | .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | 27 | .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | |
28 | .\" POSSIBILITY OF SUCH DAMAGE. | 28 | .\" POSSIBILITY OF SUCH DAMAGE. | |
29 | .\" | 29 | .\" | |
30 | .Dd January 1, 2022 | 30 | .Dd January 1, 2022 | |
31 | .Dt URL2PKG 8 | 31 | .Dt URL2PKG 8 | |
32 | .Os | 32 | .Os | |
33 | .Sh NAME | 33 | .Sh NAME | |
34 | .Nm url2pkg | 34 | .Nm url2pkg | |
35 | .Nd Automatic pkgsrc package generator | 35 | .Nd Automatic pkgsrc package generator | |
36 | .Sh SYNOPSIS | 36 | .Sh SYNOPSIS | |
37 | .Nm | 37 | .Nm | |
38 | .Op Fl v|--verbose | 38 | .Op Fl v|--verbose | |
39 | .Op Ar URL | 39 | .Ar URL | |
40 | .Sh DESCRIPTION | 40 | .Sh DESCRIPTION | |
41 | .Nm | 41 | .Nm | |
42 | takes the | 42 | takes the | |
43 | .Ar URL | 43 | .Ar URL | |
44 | to a package's distfile that should be turned into a package. | 44 | to a package's distfile that should be turned into a package. | |
45 | It fetches the distfile, extracts it, and by looking at the extracted files, | 45 | It fetches the distfile, extracts it, and by looking at the extracted files, | |
46 | fills in some variables in the package | 46 | fills in some variables in the package | |
47 | .Pa Makefile . | 47 | .Pa Makefile . | |
48 | .Pp | 48 | .Pp | |
49 | .Nm | 49 | .Nm | |
50 | helps creating a package; | 50 | helps creating a package; | |
51 | it is not intended to fly on autopilot, though. | 51 | it is not intended to fly on autopilot, though. | |
52 | .Pp | 52 | .Pp |
@@ -1,15 +1,15 @@ | @@ -1,15 +1,15 @@ | |||
1 | #! @PYTHONBIN@ | 1 | #! @PYTHONBIN@ | |
2 | # $NetBSD: url2pkg.py,v 1.34 2022/01/01 14:04:11 rillig Exp $ | 2 | # $NetBSD: url2pkg.py,v 1.35 2022/01/01 15:29:14 rillig Exp $ | |
3 | 3 | |||
4 | # Copyright (c) 2019 The NetBSD Foundation, Inc. | 4 | # Copyright (c) 2019 The NetBSD Foundation, Inc. | |
5 | # All rights reserved. | 5 | # All rights reserved. | |
6 | # | 6 | # | |
7 | # This code is derived from software contributed to The NetBSD Foundation | 7 | # This code is derived from software contributed to The NetBSD Foundation | |
8 | # by Roland Illig. | 8 | # by Roland Illig. | |
9 | # | 9 | # | |
10 | # Redistribution and use in source and binary forms, with or without | 10 | # Redistribution and use in source and binary forms, with or without | |
11 | # modification, are permitted provided that the following conditions | 11 | # modification, are permitted provided that the following conditions | |
12 | # are met: | 12 | # are met: | |
13 | # 1. Redistributions of source code must retain the above copyright | 13 | # 1. Redistributions of source code must retain the above copyright | |
14 | # notice, this list of conditions and the following disclaimer. | 14 | # notice, this list of conditions and the following disclaimer. | |
15 | # 2. Redistributions in binary form must reproduce the above copyright | 15 | # 2. Redistributions in binary form must reproduce the above copyright | |
@@ -402,26 +402,48 @@ class Generator: | @@ -402,26 +402,48 @@ class Generator: | |||
402 | ([^/?]+) # filename | 402 | ([^/?]+) # filename | |
403 | (?:\?.*)? # query parameters | 403 | (?:\?.*)? # query parameters | |
404 | $ | 404 | $ | |
405 | ''' | 405 | ''' | |
406 | m = re.search(pattern, self.url) | 406 | m = re.search(pattern, self.url) | |
407 | if not m: | 407 | if not m: | |
408 | return | 408 | return | |
409 | 409 | |||
410 | project, subdir, filename = m.groups() | 410 | project, subdir, filename = m.groups() | |
411 | self.master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/{subdir}}}' | 411 | self.master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/{subdir}}}' | |
412 | self.homepage = f'https://{project}.sourceforge.net/' | 412 | self.homepage = f'https://{project}.sourceforge.net/' | |
413 | self.distfile = filename | 413 | self.distfile = filename | |
414 | 414 | |||
415 | def adjust_site_PyPI(self): | |||
416 | pattern = r'''(?x) | |||
417 | ^ | |||
418 | https://files\.pythonhosted\.org/packages/ | |||
419 | ../../.{60}/ # hash | |||
420 | ( # distfile | |||
421 | (.*) # project | |||
422 | -[0-9].* # version | |||
423 | \.tar\.gz | |||
424 | ) | |||
425 | $ | |||
426 | ''' | |||
427 | m = re.search(pattern, self.url) | |||
428 | if not m: | |||
429 | return | |||
430 | ||||
431 | filename, project = m.groups() | |||
432 | self.master_sites = f'${{MASTER_SITE_PYPI:={project[0]}/{project}/}}' | |||
433 | self.homepage = f'https://pypi.org/project/{project}/' | |||
434 | self.distfile = filename | |||
435 | ||||
436 | ||||
415 | def adjust_site_GitHub_archive(self): | 437 | def adjust_site_GitHub_archive(self): | |
416 | pattern = r'''(?x) | 438 | pattern = r'''(?x) | |
417 | ^ | 439 | ^ | |
418 | https://github\.com/ | 440 | https://github\.com/ | |
419 | (.+)/ # org | 441 | (.+)/ # org | |
420 | (.+)/archive/ # proj | 442 | (.+)/archive/ # proj | |
421 | ((?:.+/)? # tag | 443 | ((?:.+/)? # tag | |
422 | (.+)) # distname | 444 | (.+)) # distname | |
423 | (\.tar\.gz|\.zip) # ext | 445 | (\.tar\.gz|\.zip) # ext | |
424 | $ | 446 | $ | |
425 | ''' | 447 | ''' | |
426 | m = re.search(pattern, self.url) | 448 | m = re.search(pattern, self.url) | |
427 | if not m: | 449 | if not m: | |
@@ -535,26 +557,27 @@ class Generator: | @@ -535,26 +557,27 @@ class Generator: | |||
535 | Var('#LICENSE', '=', '# TODO: (see mk/license.mk)'), | 557 | Var('#LICENSE', '=', '# TODO: (see mk/license.mk)'), | |
536 | ) | 558 | ) | |
537 | 559 | |||
538 | lines.add('# url2pkg-marker (please do not remove this line.)') | 560 | lines.add('# url2pkg-marker (please do not remove this line.)') | |
539 | lines.add('.include "../../mk/bsd.pkg.mk"') | 561 | lines.add('.include "../../mk/bsd.pkg.mk"') | |
540 | 562 | |||
541 | return lines | 563 | return lines | |
542 | 564 | |||
543 | def generate_Makefile(self) -> Lines: | 565 | def generate_Makefile(self) -> Lines: | |
544 | self.adjust_site_SourceForge() | 566 | self.adjust_site_SourceForge() | |
545 | self.adjust_site_GitHub_archive() | 567 | self.adjust_site_GitHub_archive() | |
546 | self.adjust_site_GitHub_release() | 568 | self.adjust_site_GitHub_release() | |
547 | self.foreach_site_from_sites_mk(self.adjust_site_from_sites_mk) | 569 | self.foreach_site_from_sites_mk(self.adjust_site_from_sites_mk) | |
570 | self.adjust_site_PyPI() | |||
548 | self.adjust_site_other() | 571 | self.adjust_site_other() | |
549 | self.adjust_everything_else() | 572 | self.adjust_everything_else() | |
550 | return self.generate_lines() | 573 | return self.generate_lines() | |
551 | 574 | |||
552 | def generate_package(self, g: Globals) -> Lines: | 575 | def generate_package(self, g: Globals) -> Lines: | |
553 | pkgdir = g.pkgdir | 576 | pkgdir = g.pkgdir | |
554 | makefile = pkgdir / 'Makefile' | 577 | makefile = pkgdir / 'Makefile' | |
555 | plist = pkgdir / 'PLIST' | 578 | plist = pkgdir / 'PLIST' | |
556 | 579 | |||
557 | initial_lines = self.generate_Makefile() | 580 | initial_lines = self.generate_Makefile() | |
558 | 581 | |||
559 | try: | 582 | try: | |
560 | makefile.replace(f'{makefile}.url2pkg~') | 583 | makefile.replace(f'{makefile}.url2pkg~') | |
@@ -1171,27 +1194,27 @@ class Adjuster: | @@ -1171,27 +1194,27 @@ class Adjuster: | |||
1171 | self.adjust_pkg_config() | 1194 | self.adjust_pkg_config() | |
1172 | self.adjust_po() | 1195 | self.adjust_po() | |
1173 | self.adjust_use_languages() | 1196 | self.adjust_use_languages() | |
1174 | 1197 | |||
1175 | self.generate_lines().write_to(self.g.pkgdir / 'Makefile') | 1198 | self.generate_lines().write_to(self.g.pkgdir / 'Makefile') | |
1176 | descr = (self.g.pkgdir / 'DESCR') | 1199 | descr = (self.g.pkgdir / 'DESCR') | |
1177 | descr.is_file() or Lines(*self.descr_lines).write_to(descr) | 1200 | descr.is_file() or Lines(*self.descr_lines).write_to(descr) | |
1178 | 1201 | |||
1179 | if self.regenerate_distinfo: | 1202 | if self.regenerate_distinfo: | |
1180 | self.g.bmake('distinfo') | 1203 | self.g.bmake('distinfo') | |
1181 | 1204 | |||
1182 | 1205 | |||
1183 | def usage(): | 1206 | def usage(): | |
1184 | sys.exit(f'usage: {sys.argv[0]} [-v|--verbose] [URL]') | 1207 | sys.exit(f'usage: {sys.argv[0]} [-v|--verbose] URL') | |
1185 | 1208 | |||
1186 | 1209 | |||
1187 | def main(argv: List[str], g: Globals): | 1210 | def main(argv: List[str], g: Globals): | |
1188 | if not os.path.isfile('../../mk/bsd.pkg.mk'): | 1211 | if not os.path.isfile('../../mk/bsd.pkg.mk'): | |
1189 | sys.exit(f'{argv[0]}: must be run from a package directory ' | 1212 | sys.exit(f'{argv[0]}: must be run from a package directory ' | |
1190 | f'(.../pkgsrc/category/package)') | 1213 | f'(.../pkgsrc/category/package)') | |
1191 | 1214 | |||
1192 | try: | 1215 | try: | |
1193 | opts, args = getopt.getopt(argv[1:], 'v', ['verbose']) | 1216 | opts, args = getopt.getopt(argv[1:], 'v', ['verbose']) | |
1194 | for (opt, _) in opts: | 1217 | for (opt, _) in opts: | |
1195 | if opt in ('-v', '--verbose'): | 1218 | if opt in ('-v', '--verbose'): | |
1196 | g.verbose = True | 1219 | g.verbose = True | |
1197 | except getopt.GetoptError: | 1220 | except getopt.GetoptError: |
@@ -1,14 +1,14 @@ | @@ -1,14 +1,14 @@ | |||
1 | # $NetBSD: url2pkg_test.py,v 1.35 2022/01/01 15:04:58 rillig Exp $ | 1 | # $NetBSD: url2pkg_test.py,v 1.36 2022/01/01 15:29:14 rillig Exp $ | |
2 | 2 | |||
3 | import pytest | 3 | import pytest | |
4 | from url2pkg import * | 4 | from url2pkg import * | |
5 | from textwrap import dedent | 5 | from textwrap import dedent | |
6 | 6 | |||
7 | mkcvsid = '# $''NetBSD$' | 7 | mkcvsid = '# $''NetBSD$' | |
8 | g: Globals | 8 | g: Globals | |
9 | prev_dir = Path.cwd() | 9 | prev_dir = Path.cwd() | |
10 | 10 | |||
11 | 11 | |||
12 | def setup_function(_): | 12 | def setup_function(_): | |
13 | global g | 13 | global g | |
14 | 14 | |||
@@ -603,34 +603,30 @@ def test_Generator_adjust_site_from_site | @@ -603,34 +603,30 @@ def test_Generator_adjust_site_from_site | |||
603 | def test_Generator_adjust_site_from_sites_mk__PyPI(): | 603 | def test_Generator_adjust_site_from_sites_mk__PyPI(): | |
604 | url = ('https://files.pythonhosted.org/' | 604 | url = ('https://files.pythonhosted.org/' | |
605 | + 'packages/da/8b/218264f5ce91df1ad27ce8021d51b747ef287627338fe05d170565358546/' | 605 | + 'packages/da/8b/218264f5ce91df1ad27ce8021d51b747ef287627338fe05d170565358546/' | |
606 | + 'apprise-0.9.6.tar.gz') | 606 | + 'apprise-0.9.6.tar.gz') | |
607 | generator = Generator(url) | 607 | generator = Generator(url) | |
608 | 608 | |||
609 | lines = generator.generate_Makefile() | 609 | lines = generator.generate_Makefile() | |
610 | 610 | |||
611 | assert detab(lines) == [ | 611 | assert detab(lines) == [ | |
612 | mkcvsid, | 612 | mkcvsid, | |
613 | '', | 613 | '', | |
614 | 'DISTNAME= apprise-0.9.6', | 614 | 'DISTNAME= apprise-0.9.6', | |
615 | 'CATEGORIES= pkgtools', | 615 | 'CATEGORIES= pkgtools', | |
616 | # TODO: ${MASTER_SITE_PYPI:=a/apprise/} | 616 | 'MASTER_SITES= ${MASTER_SITE_PYPI:=a/apprise/}', | |
617 | 'MASTER_SITES= https://files.pythonhosted.org/packages/da/8b/' | |||
618 | + '218264f5ce91df1ad27ce8021d51b747ef287627338fe05d170565358546/', | |||
619 | '', | 617 | '', | |
620 | 'MAINTAINER= INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org', | 618 | 'MAINTAINER= INSERT_YOUR_MAIL_ADDRESS_HERE # or use pkgsrc-users@NetBSD.org', | |
621 | # TODO: https://pypi.org/project/apprise/ | 619 | 'HOMEPAGE= https://pypi.org/project/apprise/', | |
622 | 'HOMEPAGE= https://files.pythonhosted.org/packages/da/8b/' | |||
623 | + '218264f5ce91df1ad27ce8021d51b747ef287627338fe05d170565358546/', | |||
624 | 'COMMENT= TODO: Short description of the package', | 620 | 'COMMENT= TODO: Short description of the package', | |
625 | '#LICENSE= # TODO: (see mk/license.mk)', | 621 | '#LICENSE= # TODO: (see mk/license.mk)', | |
626 | '', | 622 | '', | |
627 | '# url2pkg-marker (please do not remove this line.)', | 623 | '# url2pkg-marker (please do not remove this line.)', | |
628 | '.include "../../mk/bsd.pkg.mk"', | 624 | '.include "../../mk/bsd.pkg.mk"', | |
629 | ] | 625 | ] | |
630 | 626 | |||
631 | 627 | |||
632 | def test_Generator_adjust_site_from_sites_mk__R(tmp_path: Path): | 628 | def test_Generator_adjust_site_from_sites_mk__R(tmp_path: Path): | |
633 | g.pkgdir = tmp_path | 629 | g.pkgdir = tmp_path | |
634 | url = 'http://cran.r-project.org/src/contrib/forecast_8.7.tar.gz' | 630 | url = 'http://cran.r-project.org/src/contrib/forecast_8.7.tar.gz' | |
635 | generator = Generator(url) | 631 | generator = Generator(url) | |
636 | 632 |