Sat Jul 30 11:33:23 2022 UTC ()
lintpkgsrc: use plain text cache format

Previously, the cache format was Perl code. Now it is a list of lines,
each line containing tab-separated data.

The new format reduces the storage needs by around 50%. It also ensures
that only well-formed data is written to the cache.

The previous format contained lots of IO requests, which made loading
the cache incredibly slow.

Bump version.


(rillig)
diff -r1.40 -r1.41 pkgsrc/pkgtools/lintpkgsrc/Makefile
diff -r1.36 -r1.37 pkgsrc/pkgtools/lintpkgsrc/files/lintpkgsrc.pl

cvs diff -r1.40 -r1.41 pkgsrc/pkgtools/lintpkgsrc/Makefile (expand / switch to unified diff)

--- pkgsrc/pkgtools/lintpkgsrc/Makefile 2022/07/30 10:11:45 1.40
+++ pkgsrc/pkgtools/lintpkgsrc/Makefile 2022/07/30 11:33:23 1.41
@@ -1,17 +1,16 @@ @@ -1,17 +1,16 @@
1# $NetBSD: Makefile,v 1.40 2022/07/30 10:11:45 rillig Exp $ 1# $NetBSD: Makefile,v 1.41 2022/07/30 11:33:23 rillig Exp $
2 2
3PKGNAME= lintpkgsrc-4.98 3PKGNAME= lintpkgsrc-4.99
4PKGREVISION= 1 
5CATEGORIES= pkgtools 4CATEGORIES= pkgtools
6 5
7MAINTAINER= pkgsrc-users@NetBSD.org 6MAINTAINER= pkgsrc-users@NetBSD.org
8HOMEPAGE= https://www.NetBSD.org/Documentation/pkgsrc/ 7HOMEPAGE= https://www.NetBSD.org/Documentation/pkgsrc/
9COMMENT= Sanity checks on the complete pkgsrc tree 8COMMENT= Sanity checks on the complete pkgsrc tree
10 9
11DEPENDS+= digest>=20010101:../../pkgtools/digest 10DEPENDS+= digest>=20010101:../../pkgtools/digest
12CONFLICTS+= pkglint<4.82 11CONFLICTS+= pkglint<4.82
13 12
14USE_TOOLS+= perl:run 13USE_TOOLS+= perl:run
15 14
16WRKSRC= ${WRKDIR} 15WRKSRC= ${WRKDIR}
17USE_LANGUAGES= # none 16USE_LANGUAGES= # none

cvs diff -r1.36 -r1.37 pkgsrc/pkgtools/lintpkgsrc/files/lintpkgsrc.pl (expand / switch to unified diff)

--- pkgsrc/pkgtools/lintpkgsrc/files/lintpkgsrc.pl 2022/07/30 10:55:51 1.36
+++ pkgsrc/pkgtools/lintpkgsrc/files/lintpkgsrc.pl 2022/07/30 11:33:23 1.37
@@ -1,16 +1,16 @@ @@ -1,16 +1,16 @@
1#!@PERL5@ 1#!@PERL5@
2 2
3# $NetBSD: lintpkgsrc.pl,v 1.36 2022/07/30 10:55:51 rillig Exp $ 3# $NetBSD: lintpkgsrc.pl,v 1.37 2022/07/30 11:33:23 rillig Exp $
4 4
5# Written by David Brownlee <abs@netbsd.org>. 5# Written by David Brownlee <abs@netbsd.org>.
6# 6#
7# Caveats: 7# Caveats:
8# The 'Makefile parsing' algorithm used to obtain package versions and 8# The 'Makefile parsing' algorithm used to obtain package versions and
9# DEPENDS information is geared towards speed rather than perfection, 9# DEPENDS information is geared towards speed rather than perfection,
10# though it has gotten somewhat better over time, it only parses the 10# though it has gotten somewhat better over time, it only parses the
11# simpler Makefile conditionals. 11# simpler Makefile conditionals.
12# 12#
13# TODO: Handle fun DEPENDS like avifile-devel with 13# TODO: Handle fun DEPENDS like avifile-devel with
14# {qt2-designer>=2.2.4,qt2-designer-kde>=2.3.1nb1} 14# {qt2-designer>=2.2.4,qt2-designer-kde>=2.3.1nb1}
15 15
16use locale; 16use locale;
@@ -68,43 +68,31 @@ sub pkgver($@) { @@ -68,43 +68,31 @@ sub pkgver($@) {
68 68
69sub pkgs($@) { 69sub pkgs($@) {
70 my $self = shift; 70 my $self = shift;
71 71
72 if (@_) { 72 if (@_) {
73 return $self->{_pkgs}{$_[0]}; 73 return $self->{_pkgs}{$_[0]};
74 } else { 74 } else {
75 return (sort { $a->pkg cmp $b->pkg } values %{$self->{_pkgs}}); 75 return (sort { $a->pkg cmp $b->pkg } values %{$self->{_pkgs}});
76 } 76 }
77} 77}
78 78
79sub store($) { 79sub store($) {
80 my $self = shift; 80 my $self = shift;
81 my @pkgs = keys %{$self->{_pkgs}}; 
82 my ($cnt, $subcnt) = $self->count; 
83 81
84 print("\$pkgcnt = $cnt;\n"); 82 my $pkgs = $self->{_pkgs};
85 print("\$subpkgcnt = $subcnt;\n"); 83 foreach my $pkg (sort keys %$pkgs) {
86 map($self->{_pkgs}{$_}->store, keys %{$self->{_pkgs}}); 84 $pkgs->{$pkg}->store();
87} 85 }
88 
89sub count($) { 
90 my $self = shift; 
91 my ($pkgcnt, $pkgsubcnt); 
92 
93 map { 
94 $pkgcnt++; 
95 $pkgsubcnt += $self->{_pkgs}{$_}->count; 
96 } keys %{$self->{_pkgs}}; 
97 wantarray ? ($pkgcnt, $pkgsubcnt) : $pkgcnt; 
98} 86}
99 87
100# Pkgs is all versions of a given package (eg: apache-1.x and apache-2.x) 88# Pkgs is all versions of a given package (eg: apache-1.x and apache-2.x)
101# 89#
102package Pkgs; 90package Pkgs;
103 91
104sub add($@) { 92sub add($@) {
105 my $self = shift; 93 my $self = shift;
106 94
107 $self->{_pkgver}{$_[1]} = new PkgVer @_; 95 $self->{_pkgver}{$_[1]} = new PkgVer @_;
108} 96}
109 97
110sub new($@) { 98sub new($@) {
@@ -138,34 +126,30 @@ sub pkgver($@) { @@ -138,34 +126,30 @@ sub pkgver($@) {
138 } 126 }
139 return sort { $b->ver() cmp $a->ver() } values %{$self->{_pkgver}}; 127 return sort { $b->ver() cmp $a->ver() } values %{$self->{_pkgver}};
140} 128}
141 129
142sub latestver($) { 130sub latestver($) {
143 my $self = shift; 131 my $self = shift;
144 132
145 ($self->pkgver())[0]; 133 ($self->pkgver())[0];
146} 134}
147 135
148sub store($) { 136sub store($) {
149 my $self = shift; 137 my $self = shift;
150 138
151 print("\$pkgnum++;\n"); 139 my $pkgvers = $self->{_pkgver};
152 map($self->{_pkgver}{$_}->store, keys %{$self->{_pkgver}}); 140 foreach my $pkgver (sort keys %$pkgvers) {
153} 141 $pkgvers->{$pkgver}->store();
154 142 }
155sub count($) { 
156 my $self = shift; 
157 
158 scalar(keys %{$self->{_pkgver}}); 
159} 143}
160 144
161# PkgVer is a unique package+version 145# PkgVer is a unique package+version
162# 146#
163package PkgVer; 147package PkgVer;
164 148
165sub new($$$) { 149sub new($$$) {
166 my $class = shift; 150 my $class = shift;
167 my $self = {}; 151 my $self = {};
168 152
169 bless $self, $class; 153 bless $self, $class;
170 $self->{_pkg} = $_[0]; 154 $self->{_pkg} = $_[0];
171 $self->{_ver} = $_[1]; 155 $self->{_ver} = $_[1];
@@ -197,37 +181,39 @@ sub ver($) { @@ -197,37 +181,39 @@ sub ver($) {
197 my $self = shift; 181 my $self = shift;
198 182
199 $self->{_ver}; 183 $self->{_ver};
200} 184}
201 185
202sub vars($) { 186sub vars($) {
203 my $self = shift; 187 my $self = shift;
204 188
205 grep(!/^_(pkg|ver)$/, keys %{$self}); 189 grep(!/^_(pkg|ver)$/, keys %{$self});
206} 190}
207 191
208sub store($) { 192sub store($) {
209 my $self = shift; 193 my $self = shift;
210 my $data; 
211 
212 ($data = $self->{_pkg}) =~ s/([\\\$\@\%\"])/\\$1/g; 
213 print("\$pkgver = \$pkglist->add(\"$data\", \""); 
214 194
215 ($data = $self->{_ver}) =~ s/([\\\$\@\%\"])/\\$1/g; 195 my $name = $self->{_pkg};
216 print("$data\"); __pkgcount(1);\n"); 196 my $ver = $self->{_ver};
217 197
218 foreach ($self->vars) { 198 $name =~ /\s/ and die "cannot store package name '$name'\n";
219 ($data = $self->{$_}) =~ s/([\\\$\@\%\"])/\\$1/g; 199 $ver =~ /\s/ and die "cannot store package version '$ver'\n";
220 print("\$pkgver->var(\"$_\", \"$data\");\n"); 200 printf("package\t%s\t%s\n", $name, $ver);
 201
 202 foreach my $varname (sort $self->vars) {
 203 my $value = $self->{$varname};
 204 $varname =~ /\s/ and die "cannot store variable name '$varname'\n";
 205 $value =~ /\n/ and die "cannot store variable value '$value'\n";
 206 printf("var\t%s\t%s\n", $varname, $value);
221 } 207 }
222} 208}
223 209
224package main; 210package main;
225 211
226# Buildtime configuration 212# Buildtime configuration
227my $conf_make = '@MAKE@'; 213my $conf_make = '@MAKE@';
228my $conf_pkgsrcdir = '@PKGSRCDIR@'; 214my $conf_pkgsrcdir = '@PKGSRCDIR@';
229my $conf_prefix = '@PREFIX@'; 215my $conf_prefix = '@PREFIX@';
230my $conf_sysconfdir = '@PKG_SYSCONFDIR@'; 216my $conf_sysconfdir = '@PKG_SYSCONFDIR@';
231 217
232my ( 218my (
233 $pkglist, # list of Pkg packages 219 $pkglist, # list of Pkg packages
@@ -387,28 +373,27 @@ sub parse_eval_make_false($$) { @@ -387,28 +373,27 @@ sub parse_eval_make_false($$) {
387 373
388 $false = 0; 374 $false = 0;
389 $test = parse_expand_vars($line, $vars); 375 $test = parse_expand_vars($line, $vars);
390 376
391 # XXX This is _so_ wrong - need to parse this correctly 377 # XXX This is _so_ wrong - need to parse this correctly
392 $test =~ s/""/\r/g; 378 $test =~ s/""/\r/g;
393 $test =~ s/"//g; # " 379 $test =~ s/"//g; # "
394 $test =~ s/\r/""/g; 380 $test =~ s/\r/""/g;
395 381
396 debug("conditional: $test\n"); 382 debug("conditional: $test\n");
397 383
398 # XXX Could do something with target 384 # XXX Could do something with target
399 while ($test =~ /(target|empty|make|defined|exists)\s*\(([^()]+)\)/) { 385 while ($test =~ /(target|empty|make|defined|exists)\s*\(([^()]+)\)/) {
400 my $testname = $1; 386 my ($testname, $varname) = ($1, $2);
401 my $varname = $2; 
402 my $var; 387 my $var;
403 388
404 # Implement (some of) make's :M modifier 389 # Implement (some of) make's :M modifier
405 if ($varname =~ /^([^:]+):M(.+)$/) { 390 if ($varname =~ /^([^:]+):M(.+)$/) {
406 $varname = $1; 391 $varname = $1;
407 my $match = $2; 392 my $match = $2;
408 393
409 $var = $${vars}{$varname}; 394 $var = $${vars}{$varname};
410 $var = parse_expand_vars($var, $vars) 395 $var = parse_expand_vars($var, $vars)
411 if defined $var; 396 if defined $var;
412 397
413 $match =~ s/([{.+])/\\$1/g; 398 $match =~ s/([{.+])/\\$1/g;
414 $match =~ s/\*/.*/g; 399 $match =~ s/\*/.*/g;
@@ -1165,34 +1150,43 @@ sub parse_makefile_pkgsrc($) { @@ -1165,34 +1150,43 @@ sub parse_makefile_pkgsrc($) {
1165 1150
1166# chdir() || fail() 1151# chdir() || fail()
1167# 1152#
1168sub safe_chdir($) { 1153sub safe_chdir($) {
1169 my ($dir) = @_; 1154 my ($dir) = @_;
1170 1155
1171 debug("chdir: $dir"); 1156 debug("chdir: $dir");
1172 if (!chdir($dir)) { 1157 if (!chdir($dir)) {
1173 fail("Unable to chdir($dir): $!"); 1158 fail("Unable to chdir($dir): $!");
1174 } 1159 }
1175} 1160}
1176 1161
1177sub load_pkgsrc_makefiles($) { 1162sub load_pkgsrc_makefiles($) {
 1163 my ($fname) = @_;
1178 1164
1179 open(STORE, "<$_[0]") || die("Cannot read pkgsrc store from $_[0]: $!\n"); 1165 open(STORE, "<", $fname)
 1166 or die("Cannot read pkgsrc store from $fname: $!\n");
1180 my ($pkgver); 1167 my ($pkgver);
1181 our ($pkgcnt, $pkgnum, $subpkgcnt, $subpkgnum); 1168 $pkglist = PkgList->new;
1182 $pkglist = new PkgList; 1169 while (defined(my $line = <STORE>)) {
1183 while (<STORE>) { 1170 chomp($line);
1184 debug("eval store $_"); 1171 if ($line =~ qr"^package\t([^\t]+)\t([^\t]+$)$") {
1185 eval $_; 1172 $pkgver = $pkglist->add($1, $2);
 1173 } elsif ($line =~ qr"^var\t([^\t]+)\t(.*)$") {
 1174 $pkgver->var($1, $2);
 1175 } elsif ($line =~ qr"^sub ") {
 1176 die "Outdated cache format in '$fname'\n";
 1177 } else {
 1178 die "Invalid line '$line' in cache '$fname'\n";
 1179 }
1186 } 1180 }
1187 close(STORE); 1181 close(STORE);
1188} 1182}
1189 1183
1190# Generate pkgname->category/pkg mapping, optionally check DEPENDS 1184# Generate pkgname->category/pkg mapping, optionally check DEPENDS
1191# 1185#
1192sub scan_pkgsrc_makefiles($) { 1186sub scan_pkgsrc_makefiles($) {
1193 my ($pkgsrcdir) = @_; 1187 my ($pkgsrcdir) = @_;
1194 my (@categories); 1188 my (@categories);
1195 1189
1196 if ($pkglist) { 1190 if ($pkglist) {
1197 1191
1198 # Already done 1192 # Already done
@@ -1367,38 +1361,35 @@ sub scan_pkgsrc_distfiles_vs_distinfo($$ @@ -1367,38 +1361,35 @@ sub scan_pkgsrc_distfiles_vs_distinfo($$
1367 } 1361 }
1368 } 1362 }
1369 } 1363 }
1370 close($out); 1364 close($out);
1371 waitpid($pid, 0) || fail "xargs digest $sum"; 1365 waitpid($pid, 0) || fail "xargs digest $sum";
1372 waitpid($pid2, 0) || fail "pipe write to xargs"; 1366 waitpid($pid2, 0) || fail "pipe write to xargs";
1373 } 1367 }
1374 safe_chdir('/'); # Do not want to stay in $pkgdistdir 1368 safe_chdir('/'); # Do not want to stay in $pkgdistdir
1375 } 1369 }
1376 (sort keys %bad_distfiles); 1370 (sort keys %bad_distfiles);
1377} 1371}
1378 1372
1379sub store_pkgsrc_makefiles($) { 1373sub store_pkgsrc_makefiles($) {
1380 open(STORE, ">$_[0]") || die("Cannot save pkgsrc store to $_[0]: $!\n"); 1374 my ($fname) = @_;
1381 my $was = select(STORE); 1375
1382 print( 1376 open(STORE, ">", $fname)
1383 'sub __pkgcount { $subpkgnum += $_[0]; ', 1377 or die("Cannot save pkgsrc store to $fname: $!\n");
1384 'verbose("\rReading pkgsrc database: ', 1378 my $prev = select(STORE);
1385 '$pkgnum / $pkgcnt ($subpkgnum / $subpkgcnt) pkgs"); }', 1379 $pkglist->store();
1386 "\n" 1380 select($prev);
1387 ); 1381 close(STORE)
1388 $pkglist->store; 1382 or die("Cannot save pkgsrc store to $fname: $!\n");
1389 print("verbose(\"...done\\n\");\n"); 
1390 select($was); 
1391 close(STORE); 
1392} 1383}
1393 1384
1394# Remember to update manual page when modifying option list 1385# Remember to update manual page when modifying option list
1395# 1386#
1396sub usage_and_exit() { 1387sub usage_and_exit() {
1397 print "Usage: lintpkgsrc [opts] [makefiles] 1388 print "Usage: lintpkgsrc [opts] [makefiles]
1398opts: 1389opts:
1399 -h : This help. [see lintpkgsrc(1) for more information] 1390 -h : This help. [see lintpkgsrc(1) for more information]
1400 1391
1401Installed package options: Distfile options: 1392Installed package options: Distfile options:
1402 -i : Check version against pkgsrc -m : List distinfo mismatches 1393 -i : Check version against pkgsrc -m : List distinfo mismatches
1403 -u : As -i + fetch dist (may change) -o : List obsolete (no distinfo) 1394 -u : As -i + fetch dist (may change) -o : List obsolete (no distinfo)
1404 -y : Remove orphan distfiles 1395 -y : Remove orphan distfiles