Received: by mail.netbsd.org (Postfix, from userid 605) id 8DAF884D6D; Wed, 5 Sep 2018 17:56:50 +0000 (UTC) Received: from localhost (localhost [127.0.0.1]) by mail.netbsd.org (Postfix) with ESMTP id 909AF84D68 for ; Wed, 5 Sep 2018 17:56:49 +0000 (UTC) X-Virus-Scanned: amavisd-new at netbsd.org Received: from mail.netbsd.org ([127.0.0.1]) by localhost (mail.netbsd.org [127.0.0.1]) (amavisd-new, port 10025) with ESMTP id R6QU6k2ofKSA for ; Wed, 5 Sep 2018 17:56:23 +0000 (UTC) Received: from cvs.NetBSD.org (ivanova.NetBSD.org [IPv6:2001:470:a085:999:28c:faff:fe03:5984]) by mail.netbsd.org (Postfix) with ESMTP id 23A2484D67 for ; Wed, 5 Sep 2018 17:56:23 +0000 (UTC) Received: by cvs.NetBSD.org (Postfix, from userid 500) id 185BEFBF8; Wed, 5 Sep 2018 17:56:23 +0000 (UTC) Content-Transfer-Encoding: 7bit Content-Type: multipart/mixed; boundary="_----------=_1536170183169240" MIME-Version: 1.0 Date: Wed, 5 Sep 2018 17:56:23 +0000 From: "Roland Illig" Subject: CVS commit: pkgsrc/pkgtools/pkglint To: pkgsrc-changes@NetBSD.org Reply-To: rillig@netbsd.org X-Mailer: log_accum Message-Id: <20180905175623.185BEFBF8@cvs.NetBSD.org> Sender: pkgsrc-changes-owner@NetBSD.org List-Id: pkgsrc-changes.NetBSD.org Precedence: bulk List-Unsubscribe: This is a multi-part message in MIME format. --_----------=_1536170183169240 Content-Disposition: inline Content-Transfer-Encoding: 8bit Content-Type: text/plain; charset="US-ASCII" Module Name: pkgsrc Committed By: rillig Date: Wed Sep 5 17:56:22 UTC 2018 Modified Files: pkgsrc/pkgtools/pkglint: Makefile pkgsrc/pkgtools/pkglint/files: alternatives_test.go autofix.go autofix_test.go buildlink3_test.go category_test.go check_test.go distinfo_test.go expecter.go files_test.go licenses.go licenses_test.go line.go linechecker_test.go logging.go logging_test.go mkline.go mkline_test.go mklinechecker.go mklinechecker_test.go mklines.go mklines_test.go mklines_varalign_test.go mkparser.go mkparser_test.go mkshparser.go mkshparser_test.go mktypes_test.go options.go options_test.go package.go package_test.go parser_test.go patches.go patches_test.go pkglint.go pkglint_test.go pkgsrc.go pkgsrc_test.go plist.go plist_test.go shell.go shell_test.go shtokenizer.go shtokenizer_test.go shtypes.go shtypes_test.go substcontext.go substcontext_test.go tools.go tools_test.go util.go util_test.go vardefs.go vardefs_test.go vartype.go vartype_test.go vartypecheck.go vartypecheck_test.go pkgsrc/pkgtools/pkglint/files/getopt: getopt_test.go pkgsrc/pkgtools/pkglint/files/textproc: prefixreplacer.go pkgsrc/pkgtools/pkglint/files/trace: tracing.go Added Files: pkgsrc/pkgtools/pkglint/files/trace: tracing_test.go Log Message: pkgtools/pkglint: update to 5.6.2 Changes since 5.6.1: * Improved checks that depend on whether bsd.prefs.mk is included or not. * Improved checks for tools, whether they may be used at load time or at run time. * Improved tokenizer for shell commands. $| is not a variable but a dollar followed by a pipe. * Warnings about SUBST context are now shown by default. * A warning is shown when a SUBST block is declared for *-configure but the package has defined USE_CONFIGURE=no. * Don't warn about USE_TOOLS:= ${USE_TOOLS:Ntool}. * Don't warn about using the ?= operator in buildlink3.mk files before including bsd.prefs.mk (for some more variables, but not all). * Report an error for packages from main pkgsrc that have a TODO or README file. Packages should be simple enough that they don't need a README file and ready for production so that they don't need a TODO. * Lots of small bug fixes and new tests. To generate a diff of this commit: cvs rdiff -u -r1.547 -r1.548 pkgsrc/pkgtools/pkglint/Makefile cvs rdiff -u -r1.3 -r1.4 pkgsrc/pkgtools/pkglint/files/alternatives_test.go \ pkgsrc/pkgtools/pkglint/files/logging_test.go \ pkgsrc/pkgtools/pkglint/files/mklines_varalign_test.go \ pkgsrc/pkgtools/pkglint/files/mktypes_test.go \ pkgsrc/pkgtools/pkglint/files/options.go \ pkgsrc/pkgtools/pkglint/files/options_test.go \ pkgsrc/pkgtools/pkglint/files/tools.go \ pkgsrc/pkgtools/pkglint/files/tools_test.go cvs rdiff -u -r1.8 -r1.9 pkgsrc/pkgtools/pkglint/files/autofix.go \ pkgsrc/pkgtools/pkglint/files/parser_test.go \ pkgsrc/pkgtools/pkglint/files/pkgsrc.go \ pkgsrc/pkgtools/pkglint/files/shtypes.go cvs rdiff -u -r1.9 -r1.10 pkgsrc/pkgtools/pkglint/files/autofix_test.go \ pkgsrc/pkgtools/pkglint/files/shtokenizer.go cvs rdiff -u -r1.16 -r1.17 pkgsrc/pkgtools/pkglint/files/buildlink3_test.go \ pkgsrc/pkgtools/pkglint/files/distinfo_test.go \ pkgsrc/pkgtools/pkglint/files/files_test.go cvs rdiff -u -r1.11 -r1.12 pkgsrc/pkgtools/pkglint/files/category_test.go \ pkgsrc/pkgtools/pkglint/files/expecter.go cvs rdiff -u -r1.24 -r1.25 pkgsrc/pkgtools/pkglint/files/check_test.go \ pkgsrc/pkgtools/pkglint/files/line.go \ pkgsrc/pkgtools/pkglint/files/plist_test.go \ pkgsrc/pkgtools/pkglint/files/shell.go cvs rdiff -u -r1.13 -r1.14 pkgsrc/pkgtools/pkglint/files/licenses.go \ pkgsrc/pkgtools/pkglint/files/logging.go \ pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go \ pkgsrc/pkgtools/pkglint/files/mkparser_test.go \ pkgsrc/pkgtools/pkglint/files/substcontext_test.go cvs rdiff -u -r1.14 -r1.15 pkgsrc/pkgtools/pkglint/files/licenses_test.go cvs rdiff -u -r1.6 -r1.7 pkgsrc/pkgtools/pkglint/files/linechecker_test.go \ pkgsrc/pkgtools/pkglint/files/mkshparser.go \ pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go \ pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go \ pkgsrc/pkgtools/pkglint/files/vartype_test.go cvs rdiff -u -r1.36 -r1.37 pkgsrc/pkgtools/pkglint/files/mkline.go \ pkgsrc/pkgtools/pkglint/files/pkglint.go cvs rdiff -u -r1.40 -r1.41 pkgsrc/pkgtools/pkglint/files/mkline_test.go cvs rdiff -u -r1.17 -r1.18 pkgsrc/pkgtools/pkglint/files/mklinechecker.go cvs rdiff -u -r1.30 -r1.31 pkgsrc/pkgtools/pkglint/files/mklines.go \ pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go cvs rdiff -u -r1.26 -r1.27 pkgsrc/pkgtools/pkglint/files/mklines_test.go cvs rdiff -u -r1.15 -r1.16 pkgsrc/pkgtools/pkglint/files/mkparser.go \ pkgsrc/pkgtools/pkglint/files/vartype.go cvs rdiff -u -r1.5 -r1.6 pkgsrc/pkgtools/pkglint/files/mkshparser_test.go cvs rdiff -u -r1.34 -r1.35 pkgsrc/pkgtools/pkglint/files/package.go cvs rdiff -u -r1.28 -r1.29 pkgsrc/pkgtools/pkglint/files/package_test.go cvs rdiff -u -r1.22 -r1.23 pkgsrc/pkgtools/pkglint/files/patches.go cvs rdiff -u -r1.19 -r1.20 pkgsrc/pkgtools/pkglint/files/patches_test.go cvs rdiff -u -r1.23 -r1.24 pkgsrc/pkgtools/pkglint/files/pkglint_test.go cvs rdiff -u -r1.27 -r1.28 pkgsrc/pkgtools/pkglint/files/plist.go \ pkgsrc/pkgtools/pkglint/files/util.go cvs rdiff -u -r1.29 -r1.30 pkgsrc/pkgtools/pkglint/files/shell_test.go cvs rdiff -u -r1.4 -r1.5 pkgsrc/pkgtools/pkglint/files/shtypes_test.go cvs rdiff -u -r1.12 -r1.13 pkgsrc/pkgtools/pkglint/files/substcontext.go \ pkgsrc/pkgtools/pkglint/files/util_test.go cvs rdiff -u -r1.44 -r1.45 pkgsrc/pkgtools/pkglint/files/vardefs.go cvs rdiff -u -r1.2 -r1.3 pkgsrc/pkgtools/pkglint/files/vardefs_test.go cvs rdiff -u -r1.38 -r1.39 pkgsrc/pkgtools/pkglint/files/vartypecheck.go cvs rdiff -u -r1.4 -r1.5 pkgsrc/pkgtools/pkglint/files/getopt/getopt_test.go cvs rdiff -u -r1.4 -r1.5 \ pkgsrc/pkgtools/pkglint/files/textproc/prefixreplacer.go cvs rdiff -u -r1.1 -r1.2 pkgsrc/pkgtools/pkglint/files/trace/tracing.go cvs rdiff -u -r0 -r1.1 pkgsrc/pkgtools/pkglint/files/trace/tracing_test.go Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. --_----------=_1536170183169240 Content-Disposition: inline Content-Length: 325751 Content-Transfer-Encoding: binary Content-Type: text/x-diff; charset=us-ascii Modified files: Index: pkgsrc/pkgtools/pkglint/Makefile diff -u pkgsrc/pkgtools/pkglint/Makefile:1.547 pkgsrc/pkgtools/pkglint/Makefile:1.548 --- pkgsrc/pkgtools/pkglint/Makefile:1.547 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/Makefile Wed Sep 5 17:56:22 2018 @@ -1,6 +1,6 @@ -# $NetBSD: Makefile,v 1.547 2018/08/16 20:41:42 rillig Exp $ +# $NetBSD: Makefile,v 1.548 2018/09/05 17:56:22 rillig Exp $ -PKGNAME= pkglint-5.6.1 +PKGNAME= pkglint-5.6.2 DISTFILES= # none CATEGORIES= pkgtools Index: pkgsrc/pkgtools/pkglint/files/alternatives_test.go diff -u pkgsrc/pkgtools/pkglint/files/alternatives_test.go:1.3 pkgsrc/pkgtools/pkglint/files/alternatives_test.go:1.4 --- pkgsrc/pkgtools/pkglint/files/alternatives_test.go:1.3 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/alternatives_test.go Wed Sep 5 17:56:22 2018 @@ -10,7 +10,8 @@ func (s *Suite) Test_Alternatives_PLIST( "sbin/sendmail @PREFIX@/sbin/sendmail.postfix@POSTFIXVER@", "sbin/sendmail @PREFIX@/sbin/sendmail.exim@EXIMVER@", "bin/echo bin/gnu-echo", - "bin/editor bin/vim -e") + "bin/editor bin/vim -e", + "invalid") G.Pkg = NewPackage(".") G.Pkg.PlistFiles["bin/echo"] = true @@ -24,5 +25,20 @@ func (s *Suite) Test_Alternatives_PLIST( "NOTE: ALTERNATIVES:1: @PREFIX@/ can be omitted from the file name.", "NOTE: ALTERNATIVES:2: @PREFIX@/ can be omitted from the file name.", "ERROR: ALTERNATIVES:3: Alternative wrapper \"bin/echo\" must not appear in the PLIST.", - "ERROR: ALTERNATIVES:3: Alternative implementation \"bin/gnu-echo\" must appear in the PLIST.") + "ERROR: ALTERNATIVES:3: Alternative implementation \"bin/gnu-echo\" must appear in the PLIST.", + "ERROR: ALTERNATIVES:5: Invalid ALTERNATIVES line \"invalid\".") +} + +func (s *Suite) Test_CheckfileAlternatives__empty(c *check.C) { + t := s.Init(c) + + t.Chdir("category/package") + t.SetupFileLines("ALTERNATIVES") + + G.Pkg = NewPackage(".") + + CheckfileAlternatives("ALTERNATIVES", G.Pkg.PlistFiles) + + t.CheckOutputLines( + "ERROR: ALTERNATIVES: Must not be empty.") } Index: pkgsrc/pkgtools/pkglint/files/logging_test.go diff -u pkgsrc/pkgtools/pkglint/files/logging_test.go:1.3 pkgsrc/pkgtools/pkglint/files/logging_test.go:1.4 --- pkgsrc/pkgtools/pkglint/files/logging_test.go:1.3 Sun Mar 4 20:34:33 2018 +++ pkgsrc/pkgtools/pkglint/files/logging_test.go Wed Sep 5 17:56:22 2018 @@ -179,3 +179,63 @@ func (s *Suite) Test_explain_with_only(c "\tThis explanation is logged.", "") } + +func (s *Suite) Test_logs__duplicate_messages(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("--explain") + G.opts.LogVerbose = false + line := t.NewLine("README.txt", 123, "text") + + // In rare cases, the explanations for the same warning may differ + // when they appear in different contexts. In such a case, if the + // warning is suppressed, the explanation must not appear on its own. + line.Warnf("The warning.") // Is logged + Explain("Explanation 1") + line.Warnf("The warning.") // Is suppressed + Explain("Explanation 2") + + t.CheckOutputLines( + "WARN: README.txt:123: The warning.", + "", + "\tExplanation 1", + "") +} + +func (s *Suite) Test_logs__duplicate_explanations(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("--explain") + line := t.NewLine("README.txt", 123, "text") + + // In rare cases, different diagnostics may have the same explanation. + line.Warnf("Warning 1.") + Explain("Explanation") + line.Warnf("Warning 2.") + Explain("Explanation") // Is suppressed. + + t.CheckOutputLines( + "WARN: README.txt:123: Warning 1.", + "", + "\tExplanation", + "", + "WARN: README.txt:123: Warning 2.") +} + +func (s *Suite) Test_logs__panic(c *check.C) { + c.Check(func() { + logs(llError, "filename", "13", "No period", "No period") + }, check.Panics, "Diagnostic format \"No period\" must end in a period.") +} + +func (s *Suite) Test_Explain__long_lines(c *check.C) { + t := s.Init(c) + + Explain( + "123456789 12345678. abcdefghi. 123456789 123456789 123456789 123456789 123456789 ") + + t.CheckOutputLines( + "Long explanation line: 123456789 12345678. abcdefghi. 123456789 123456789 123456789 123456789 123456789 ", + "Break after: 123456789 12345678. abcdefghi. 123456789 123456789 123456789", + "Short space after period: 123456789 12345678. abcdefghi. 123456789 123456789 123456789 123456789 123456789 ") +} Index: pkgsrc/pkgtools/pkglint/files/mklines_varalign_test.go diff -u pkgsrc/pkgtools/pkglint/files/mklines_varalign_test.go:1.3 pkgsrc/pkgtools/pkglint/files/mklines_varalign_test.go:1.4 --- pkgsrc/pkgtools/pkglint/files/mklines_varalign_test.go:1.3 Thu Aug 9 20:08:12 2018 +++ pkgsrc/pkgtools/pkglint/files/mklines_varalign_test.go Wed Sep 5 17:56:22 2018 @@ -41,30 +41,15 @@ func (vt *VaralignTester) Fixed(lines .. // Run is called after setting up the data and runs the varalign checks twice. // Once for getting the diagnostics and once for automatically fixing them. func (vt *VaralignTester) Run() { - vt.runDefault() - vt.runAutofix() + vt.run(false) + vt.run(true) } -func (vt *VaralignTester) runDefault() { +func (vt *VaralignTester) run(autofix bool) { cmdline := []string{"-Wall"} - if vt.source { - cmdline = append(cmdline, "--source") + if autofix { + cmdline = append(cmdline, "--autofix") } - vt.tester.SetupCommandLine(cmdline...) - - mklines := vt.tester.SetupFileMkLines("Makefile", vt.input...) - - varalign := VaralignBlock{} - for _, mkline := range mklines.mklines { - varalign.Check(mkline) - } - varalign.Finish() - - vt.tester.CheckOutputLines(vt.diagnostics...) -} - -func (vt *VaralignTester) runAutofix() { - cmdline := []string{"-Wall", "--autofix"} if vt.source { cmdline = append(cmdline, "--source") } @@ -78,10 +63,14 @@ func (vt *VaralignTester) runAutofix() { } varalign.Finish() - vt.tester.CheckOutputLines(vt.autofixes...) + if autofix { + vt.tester.CheckOutputLines(vt.autofixes...) - SaveAutofixChanges(mklines.lines) - vt.tester.CheckFileLinesDetab("Makefile", vt.fixed...) + SaveAutofixChanges(mklines.lines) + vt.tester.CheckFileLinesDetab("Makefile", vt.fixed...) + } else { + vt.tester.CheckOutputLines(vt.diagnostics...) + } } // Generally, the value in variable assignments is aligned @@ -978,3 +967,21 @@ func (s *Suite) Test_Varalign__realign_c "# file2") vt.Run() } + +// FIXME: The diagnostic does not correspond to the autofix; see "if oldWidth == 8". +func (s *Suite) Test_Varalign__mixed_indentation(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR1=\tvalue1", + "VAR2=\tvalue2 \\", + " \t \t value2 continued") + vt.Diagnostics( + /*"NOTE: ~/Makefile:2--3: This line should be aligned with \"\\t\"."*/ ) + vt.Autofixes( + /*"AUTOFIX: ~/Makefile:3: Replacing indentation \" \\t \\t \" with \"\\t\\t \"."*/ ) + vt.Fixed( + "VAR1= value1", + "VAR2= value2 \\", + " value2 continued") + vt.Run() +} Index: pkgsrc/pkgtools/pkglint/files/mktypes_test.go diff -u pkgsrc/pkgtools/pkglint/files/mktypes_test.go:1.3 pkgsrc/pkgtools/pkglint/files/mktypes_test.go:1.4 --- pkgsrc/pkgtools/pkglint/files/mktypes_test.go:1.3 Sun Jul 10 21:24:47 2016 +++ pkgsrc/pkgtools/pkglint/files/mktypes_test.go Wed Sep 5 17:56:22 2018 @@ -1,7 +1,7 @@ package main import ( - check "gopkg.in/check.v1" + "gopkg.in/check.v1" ) func NewMkVarUse(varname string, modifiers ...string) *MkVarUse { Index: pkgsrc/pkgtools/pkglint/files/options.go diff -u pkgsrc/pkgtools/pkglint/files/options.go:1.3 pkgsrc/pkgtools/pkglint/files/options.go:1.4 --- pkgsrc/pkgtools/pkglint/files/options.go:1.3 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/options.go Wed Sep 5 17:56:22 2018 @@ -53,12 +53,10 @@ loop: switch { case matches(includedFile, `/[^/]+\.buildlink3\.mk$`): case matches(includedFile, `/[^/]+\.builtin\.mk$`): - case includedFile == "../../mk/bsd.prefs.mk": - case includedFile == "../../mk/bsd.fast.prefs.mk": - case includedFile == "../../mk/bsd.options.mk": exp.Advance() break loop + case IsPrefs(includedFile): } default: Index: pkgsrc/pkgtools/pkglint/files/options_test.go diff -u pkgsrc/pkgtools/pkglint/files/options_test.go:1.3 pkgsrc/pkgtools/pkglint/files/options_test.go:1.4 --- pkgsrc/pkgtools/pkglint/files/options_test.go:1.3 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/options_test.go Wed Sep 5 17:56:22 2018 @@ -5,7 +5,7 @@ import "gopkg.in/check.v1" func (s *Suite) Test_ChecklinesOptionsMk(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wno-space") + t.SetupCommandLine("-Wall,no-space") t.SetupVartypes() t.SetupOption("mc-charset", "") t.SetupOption("mysql", "") @@ -31,6 +31,8 @@ func (s *Suite) Test_ChecklinesOptionsMk "", ".include \"../../mk/bsd.options.mk\"", "", + "PKGNAME?= default-pkgname-1.", + "", ".if !empty(PKG_OPTIONS:Mx11)", ".endif", "", @@ -52,10 +54,10 @@ func (s *Suite) Test_ChecklinesOptionsMk ChecklinesOptionsMk(mklines) t.CheckOutputLines( - "WARN: ~/category/package/options.mk:16: Unknown option \"undeclared\".", - "NOTE: ~/category/package/options.mk:19: The positive branch of the .if/.else should be the one where the option is set.", + "WARN: ~/category/package/options.mk:18: Unknown option \"undeclared\".", + "NOTE: ~/category/package/options.mk:21: The positive branch of the .if/.else should be the one where the option is set.", "WARN: ~/category/package/options.mk:6: Option \"mc-charset\" should be handled below in an .if block.", - "WARN: ~/category/package/options.mk:16: Option \"undeclared\" is handled but not added to PKG_SUPPORTED_OPTIONS.") + "WARN: ~/category/package/options.mk:18: Option \"undeclared\" is handled but not added to PKG_SUPPORTED_OPTIONS.") } func (s *Suite) Test_ChecklinesOptionsMk__unexpected_line(c *check.C) { Index: pkgsrc/pkgtools/pkglint/files/tools.go diff -u pkgsrc/pkgtools/pkglint/files/tools.go:1.3 pkgsrc/pkgtools/pkglint/files/tools.go:1.4 --- pkgsrc/pkgtools/pkglint/files/tools.go:1.3 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/tools.go Wed Sep 5 17:56:22 2018 @@ -2,6 +2,7 @@ package main import ( "netbsd.org/pkglint/trace" + "path" "sort" "strings" ) @@ -12,73 +13,140 @@ import ( // // See `mk/tools/`. type Tool struct { - Name string // e.g. "sed", "gzip" - Varname string // e.g. "SED", "GZIP_CMD" - MustUseVarForm bool // True for `echo`, because of many differing implementations. - Predefined bool // This tool is used by the pkgsrc infrastructure, therefore the package does not need to add it to `USE_TOOLS` explicitly. - UsableAtLoadTime bool // May be used after including `bsd.prefs.mk`. -} - -type ToolRegistry struct { - byName map[string]*Tool - byVarname map[string]*Tool -} - -func NewToolRegistry() ToolRegistry { - return ToolRegistry{make(map[string]*Tool), make(map[string]*Tool)} -} - -// Register registers the tool by its name. -// The tool may then be used by this name (e.g. "awk"), -// but not by a corresponding variable (e.g. ${AWK}). -// The toolname may include the scope (:pkgsrc, :run, etc.). -func (tr *ToolRegistry) Register(toolname string, mkline MkLine) *Tool { - name := strings.SplitN(toolname, ":", 2)[0] - tr.validateToolName(name, mkline) - - tool := tr.byName[name] - if tool == nil { - tool = &Tool{Name: name} - tr.byName[name] = tool + Name string // e.g. "sed", "gzip" + Varname string // e.g. "SED", "GZIP_CMD" + MustUseVarForm bool // True for `echo`, because of many differing implementations. + Validity Validity +} + +func (tool *Tool) SetValidity(validity Validity, traceName string) { + if trace.Tracing && validity != tool.Validity { + trace.Stepf("%s: Setting validity of %q to %s", traceName, tool.Name, validity) } - return tool + tool.Validity = validity } -func (tr *ToolRegistry) RegisterVarname(toolname, varname string, mkline MkLine) *Tool { - tool := tr.Register(toolname, mkline) - tool.Varname = varname - tr.byVarname[varname] = tool - return tool +// UsableAtLoadTime means that the tool may be used by its variable +// name after bsd.prefs.mk has been included. +// +// Additionally, all allowed cases from UsableAtRunTime are allowed. +// +// VAR:= ${TOOL} # Not allowed since bsd.prefs.mk is not +// # included yet. +// +// .include "../../bsd.prefs.mk" +// +// VAR:= ${TOOL} # Allowed. +// VAR!= ${TOOL} # Allowed. +// +// VAR= ${${TOOL}:sh} # Allowed; the :sh modifier is evaluated +// # lazily, but when VAR should ever be +// # evaluated at load time, this still means +// # load time. +// +// .if ${TOOL:T} == "tool" # Allowed. +// .endif +func (tool *Tool) UsableAtLoadTime(seenPrefs bool) bool { + return seenPrefs && tool.Validity == AfterPrefsMk } -func (tr *ToolRegistry) RegisterTool(tool *Tool, mkline MkLine) { - tr.validateToolName(tool.Name, mkline) +// UsableAtRunTime means that the tool may be used by its simple name +// in all {pre,do,post}-* targets, and by its variable name in all +// runtime contexts. +// +// VAR:= ${TOOL} # Not allowed; TOOL might not be initialized yet. +// VAR!= ${TOOL} # Not allowed; TOOL might not be initialized yet. +// +// VAR= ${${TOOL}:sh} # Probably ok; the :sh modifier is evaluated at +// # run time. But if VAR should ever be evaluated +// # at load time (see the "Not allowed" cases +// # above), it doesn't work. Currently pkglint +// # cannot detect these cases reliably. +// +// own-target: +// ${TOOL} # Allowed. +// tool # Not allowed because the PATH might not be set +// # up for this target. +// +// pre-configure: +// ${TOOL} # Allowed. +// tool # Allowed. +func (tool *Tool) UsableAtRunTime() bool { + return tool.Validity == AtRunTime || tool.Validity == AfterPrefsMk +} + +// Tools collects all tools for a certain scope (global or file) +// and remembers whether these tools are defined at all, +// and whether they are declared to be used via USE_TOOLS. +type Tools struct { + TraceName string // Only for the trace log + byName map[string]*Tool // "sed" => tool + byVarname map[string]*Tool // "GREP_CMD" => tool + SeenPrefs bool // Determines the effect of adding the tool to USE_TOOLS +} + +func NewTools(traceName string) Tools { + return Tools{ + traceName, + make(map[string]*Tool), + make(map[string]*Tool), + false} +} - if tool.Name != "" && tr.byName[tool.Name] == nil { - tr.byName[tool.Name] = tool +// Define registers the tool by its name and the corresponding +// variable name (if nonempty). +// +// After this tool is added to USE_TOOLS, it may be used by this name +// (e.g. "awk") or by its variable (e.g. ${AWK}). +func (tr *Tools) Define(name, varname string, mkline MkLine) *Tool { + if trace.Tracing { + trace.Stepf("Tools.Define for %s: %q %q in %s", tr.TraceName, name, varname, mkline) } - if tool.Varname != "" && tr.byVarname[tool.Varname] == nil { - tr.byVarname[tool.Varname] = tool + + tool := tr.def(name, varname, mkline) + if varname != "" { + tool.Varname = varname } + return tool } -func (tr *ToolRegistry) FindByCommand(cmd *ShToken) *Tool { - if tool := tr.byName[cmd.MkText]; tool != nil { - return tool +func (tr *Tools) def(name, varname string, mkline MkLine) *Tool { + if mkline != nil && !tr.IsValidToolName(name) { + mkline.Errorf("Invalid tool name %q.", name) } - if len(cmd.Atoms) == 1 { - if varuse := cmd.Atoms[0].VarUse(); varuse != nil { - if tool := tr.byVarname[varuse.varname]; tool != nil { - return tool - } + + validity := Nowhere + if mkline != nil { + if IsPrefs(mkline.Filename) { + validity = AfterPrefsMk + } else if path.Base(mkline.Filename) == "bsd.pkg.mk" { + validity = AtRunTime + } + } + tool := &Tool{name, varname, false, validity} + + if name != "" { + if existing := tr.byName[name]; existing != nil { + tool = existing + } else { + tr.byName[name] = tool } } - return nil + + if varname != "" { + if existing := tr.byVarname[varname]; existing == nil || len(existing.Name) > len(name) { + tr.byVarname[varname] = tool + } + } + + return tool } -func (tr *ToolRegistry) Trace() { +func (tr *Tools) Trace() { if trace.Tracing { - defer trace.Call0()() + defer trace.Call1(tr.TraceName)() + } else { + return } var keys []string @@ -92,46 +160,178 @@ func (tr *ToolRegistry) Trace() { } } -// ParseToolLine parses a tool definition from the pkgsrc infrastructure, -// e.g. in mk/tools/replace.mk. -func (tr *ToolRegistry) ParseToolLine(mkline MkLine) { - if mkline.IsVarassign() { - varname := mkline.Varname() +// ParseToolLine updates the tool definitions according to the given +// line from a Makefile. +func (tr *Tools) ParseToolLine(mkline MkLine) { + tr.ParseToolLineCreate(mkline, false) +} + +// ParseToolLineCreate updates the tool definitions according to the given +// line from a Makefile, registering the tools if necessary. +func (tr *Tools) ParseToolLineCreate(mkline MkLine, createIfAbsent bool) { + switch { + + case mkline.IsVarassign(): + varparam := mkline.Varparam() value := mkline.Value() - if varname == "TOOLS_CREATE" && (value == "[" || matches(value, `^?[-\w.]+$`)) { - tr.Register(value, mkline) - } else if m, toolname := match1(varname, `^_TOOLS_VARNAME\.([-\w.]+|\[)$`); m { - tr.RegisterVarname(toolname, value, mkline) + switch mkline.Varcanon() { + case "TOOLS_CREATE": + if tr.IsValidToolName(value) { + tr.Define(value, "", mkline) + } - } else if m, toolname = match1(varname, `^(?:TOOLS_PATH|_TOOLS_DEPMETHOD)\.([-\w.]+|\[)$`); m { - tr.Register(toolname, mkline) + case "_TOOLS_VARNAME.*": + if !containsVarRef(varparam) { + tr.Define(varparam, value, mkline) + } + + case "TOOLS_PATH.*", "_TOOLS_DEPMETHOD.*": + if !containsVarRef(varparam) { + tr.Define(varparam, "", mkline) + } - } else if m, toolname = match1(varname, `^_TOOLS\.(.*)`); m { - tr.Register(toolname, mkline) - for _, tool := range splitOnSpace(value) { - tr.Register(tool, mkline) + case "_TOOLS.*": + if !containsVarRef(varparam) { + tr.Define(varparam, "", mkline) + for _, tool := range splitOnSpace(value) { + tr.Define(tool, "", mkline) + } } + + case "USE_TOOLS": + tr.parseUseTools(mkline, createIfAbsent) + } + + case mkline.IsInclude(): + if IsPrefs(mkline.IncludeFile()) { + tr.SeenPrefs = true } } } -func (tr *ToolRegistry) ByVarname(varname string) *Tool { - return tr.byVarname[varname] +// parseUseTools interprets a "USE_TOOLS+=" line from a Makefile fragment. +// It determines the validity of the tool, i.e. in which places it may be used. +// +// If createIfAbsent is true and the tools is unknown, it is registered. +func (tr *Tools) parseUseTools(mkline MkLine, createIfAbsent bool) { + value := mkline.Value() + if containsVarRef(value) { + return + } + + deps := splitOnSpace(value) + + // See mk/tools/autoconf.mk:/^\.if !defined/ + if matches(value, `\bautoconf213\b`) { + for _, name := range [...]string{"autoconf-2.13", "autoheader-2.13", "autoreconf-2.13", "autoscan-2.13", "autoupdate-2.13", "ifnames-2.13"} { + if createIfAbsent { + tr.Define(name, "", mkline) + } + deps = append(deps, name) + } + } + if matches(value, `\bautoconf\b`) { + for _, name := range [...]string{"autoheader", "autom4te", "autoreconf", "autoscan", "autoupdate", "ifnames"} { + if createIfAbsent { + tr.Define(name, "", mkline) + } + deps = append(deps, name) + } + } + + for _, dep := range deps { + name := strings.Split(dep, ":")[0] + tool := tr.ByName(name) + if tool == nil && createIfAbsent { + tr.Define(name, "", mkline) + } + if tool != nil { + validity := tr.validity(mkline.Filename) + if validity > tool.Validity { + tool.SetValidity(validity, tr.TraceName) + } + } + } } -func (tr *ToolRegistry) ByName(name string) *Tool { - return tr.byName[name] +func (tr *Tools) validity(fileName string) Validity { + basename := path.Base(fileName) + if basename == "Makefile" && tr.SeenPrefs { + return AtRunTime + } + if basename == "bsd.prefs.mk" || basename == "Makefile" { + return AfterPrefsMk + } + return AtRunTime } -func (tr *ToolRegistry) ForEach(action func(tool *Tool)) { - for _, tool := range tr.byName { - action(tool) +func (tr *Tools) ByVarname(varname string) (tool *Tool) { return tr.byVarname[varname] } + +func (tr *Tools) ByName(name string) (tool *Tool) { return tr.byName[name] } + +func (tr *Tools) Usable(tool *Tool, time ToolTime) bool { + if time == LoadTime { + return tool.UsableAtLoadTime(tr.SeenPrefs) + } else { + return tool.UsableAtRunTime() } } -func (tr *ToolRegistry) validateToolName(toolName string, mkline MkLine) { - if toolName != "echo -n" && !matches(toolName, `^([-a-z0-9.]+|\[)$`) { - mkline.Errorf("Invalid tool name %q.", toolName) +func (tr *Tools) AddAll(other Tools) { + if trace.Tracing { + defer trace.Call(other.TraceName, "to", tr.TraceName)() } + + for _, otherTool := range other.byName { + if trace.Tracing { + trace.Stepf("Tools.AddAll %+v", *otherTool) + } + tool := tr.def(otherTool.Name, otherTool.Varname, nil) + tool.MustUseVarForm = tool.MustUseVarForm || otherTool.MustUseVarForm + if otherTool.Validity > tool.Validity { + tool.SetValidity(otherTool.Validity, tr.TraceName) + } + } +} + +func (tr *Tools) IsValidToolName(name string) bool { + return name == "[" || name == "echo -n" || matches(name, `^[-0-9a-z.]+$`) } + +type Validity uint8 + +const ( + // Nowhere means that the tool has not been added + // to USE_TOOLS and therefore cannot be used at all. + Nowhere Validity = iota + + // AtRunTime means that the tool has been added to USE_TOOLS + // after including bsd.prefs.mk and therefore cannot be used + // at load time. + // + // The tool may be used as ${TOOL} in all targets. + // The tool may be used by its plain name in {pre,do,post}-* targets. + AtRunTime + + // AfterPrefsMk means that the tool has been added to USE_TOOLS + // before including bsd.prefs.mk and therefore can be used at + // load time after bsd.prefs.mk has been included. + // + // The tool may be used as ${TOOL} everywhere. + // The tool may be used by its plain name in {pre,do,post}-* targets. + AfterPrefsMk +) + +func (time Validity) String() string { + return [...]string{"Nowhere", "AtRunTime", "AfterPrefsMk"}[time] +} + +type ToolTime uint8 + +const ( + LoadTime ToolTime = iota + RunTime +) + +func (t ToolTime) String() string { return [...]string{"LoadTime", "RunTime"}[t] } Index: pkgsrc/pkgtools/pkglint/files/tools_test.go diff -u pkgsrc/pkgtools/pkglint/files/tools_test.go:1.3 pkgsrc/pkgtools/pkglint/files/tools_test.go:1.4 --- pkgsrc/pkgtools/pkglint/files/tools_test.go:1.3 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/tools_test.go Wed Sep 5 17:56:22 2018 @@ -2,10 +2,10 @@ package main import "gopkg.in/check.v1" -func (s *Suite) Test_ToolRegistry_ParseToolLine(c *check.C) { +func (s *Suite) Test_Tools_ParseToolLine(c *check.C) { t := s.Init(c) - t.SetupTool(&Tool{Name: "tool1", Predefined: true}) + t.SetupToolUsable("tool1", "") t.SetupVartypes() t.SetupFileLines("Makefile", MkRcsID, @@ -18,15 +18,421 @@ func (s *Suite) Test_ToolRegistry_ParseT t.CheckOutputEmpty() } -func (s *Suite) Test_ToolRegistry_validateToolName__invalid(c *check.C) { +func (s *Suite) Test_Tools_validateToolName__invalid(c *check.C) { t := s.Init(c) - reg := NewToolRegistry() + reg := NewTools("") - reg.Register("tool_name", dummyMkLine) + reg.Define("tool_name", "", dummyMkLine) + reg.Define("tool:dependency", "", dummyMkLine) + reg.Define("tool:build", "", dummyMkLine) // Currently, the underscore is not used in any tool name. - // If there should ever be such a case, just use a different character. + // If there should ever be such a case, just use a different character for testing. t.CheckOutputLines( - "ERROR: Invalid tool name \"tool_name\".") + "ERROR: Invalid tool name \"tool_name\".", + "ERROR: Invalid tool name \"tool:dependency\".", + "ERROR: Invalid tool name \"tool:build\".") +} + +func (s *Suite) Test_Tools_Trace__coverage(c *check.C) { + t := s.Init(c) + + t.DisableTracing() + + reg := NewTools("") + reg.Trace() + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Tools__USE_TOOLS_predefined_sed(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.CreateFileLines("mk/bsd.prefs.mk", + "USE_TOOLS+=\tsed:pkgsrc") + t.CreateFileLines("mk/tools/defaults.mk", + "_TOOLS_VARNAME.sed=\tSED") + t.SetupFileMkLines("module.mk", + MkRcsID, + "", + "do-build:", + "\t${SED} < input > output", + "\t${AWK} < input > output") + + G.Main("pkglint", "-Wall", t.File("module.mk")) + + t.CheckOutputLines( + "WARN: ~/module.mk:5: Unknown shell command \"${AWK}\".", + "0 errors and 1 warning found.", + "(Run \"pkglint -e\" to show explanations.)") +} + +func (s *Suite) Test_Tools__add_varname_later(c *check.C) { + + tools := NewTools("") + tool := tools.Define("tool", "", dummyMkLine) + + c.Check(tool.Name, equals, "tool") + c.Check(tool.Varname, equals, "") + + // Should update the existing tool definition. + tools.Define("tool", "TOOL", dummyMkLine) + + c.Check(tool.Name, equals, "tool") + c.Check(tool.Varname, equals, "TOOL") +} + +func (s *Suite) Test_Tools__load_from_infrastructure(c *check.C) { + t := s.Init(c) + + tools := NewTools("") + + tools.ParseToolLine(t.NewMkLine("create.mk", 2, "TOOLS_CREATE+= load")) + tools.ParseToolLine(t.NewMkLine("create.mk", 3, "TOOLS_CREATE+= run")) + tools.ParseToolLine(t.NewMkLine("create.mk", 4, "TOOLS_CREATE+= nowhere")) + + // The references to the tools are stable, + // the lookup methods always return the same objects. + load := tools.ByName("load") + run := tools.ByName("run") + nowhere := tools.ByName("nowhere") + + // All tools are defined by name, but their variable names are not yet known. + // At this point they may not be used, neither by the pkgsrc infrastructure nor by a package. + c.Check(load, deepEquals, &Tool{"load", "", false, Nowhere}) + c.Check(run, deepEquals, &Tool{"run", "", false, Nowhere}) + c.Check(nowhere, deepEquals, &Tool{"nowhere", "", false, Nowhere}) + + // The name RUN_CMD avoids conflicts with RUN. + tools.ParseToolLine(t.NewMkLine("varnames.mk", 2, "_TOOLS_VARNAME.load= LOAD")) + tools.ParseToolLine(t.NewMkLine("varnames.mk", 3, "_TOOLS_VARNAME.run= RUN_CMD")) + tools.ParseToolLine(t.NewMkLine("varnames.mk", 4, "_TOOLS_VARNAME.nowhere= NOWHERE")) + + // At this point the tools can be found by their variable names, too. + // They still may not be used. + c.Check(load, deepEquals, &Tool{"load", "LOAD", false, Nowhere}) + c.Check(run, deepEquals, &Tool{"run", "RUN_CMD", false, Nowhere}) + c.Check(nowhere, deepEquals, &Tool{"nowhere", "NOWHERE", false, Nowhere}) + c.Check(tools.ByVarname("LOAD"), equals, load) + c.Check(tools.ByVarname("RUN_CMD"), equals, run) + c.Check(tools.ByVarname("NOWHERE"), equals, nowhere) + c.Check(load.UsableAtLoadTime(false), equals, false) + c.Check(load.UsableAtLoadTime(true), equals, false) + c.Check(load.UsableAtRunTime(), equals, false) + c.Check(run.UsableAtLoadTime(false), equals, false) + c.Check(run.UsableAtLoadTime(true), equals, false) + c.Check(run.UsableAtRunTime(), equals, false) + c.Check(nowhere.UsableAtLoadTime(false), equals, false) + c.Check(nowhere.UsableAtLoadTime(true), equals, false) + c.Check(nowhere.UsableAtRunTime(), equals, false) + + tools.ParseToolLine(t.NewMkLine("bsd.prefs.mk", 2, "USE_TOOLS+= load")) + + // Tools that are added to USE_TOOLS in bsd.prefs.mk may be used afterwards. + // By variable name, they may be used both at load time as well as run time. + // By plain name, they may be used only in {pre,do,post}-* targets. + c.Check(load, deepEquals, &Tool{"load", "LOAD", false, AfterPrefsMk}) + c.Check(load.UsableAtLoadTime(false), equals, false) + c.Check(load.UsableAtLoadTime(true), equals, true) + c.Check(load.UsableAtRunTime(), equals, true) + + tools.ParseToolLine(t.NewMkLine("bsd.pkg.mk", 2, "USE_TOOLS+= run")) + + // Tools that are added to USE_TOOLS in bsd.pkg.mk may be used afterwards at run time. + // The {pre,do,post}-* targets may use both forms (${CAT} and cat). + // All other targets must use the variable form (${CAT}). + c.Check(run, deepEquals, &Tool{"run", "RUN_CMD", false, AtRunTime}) + c.Check(run.UsableAtLoadTime(false), equals, false) + c.Check(run.UsableAtLoadTime(false), equals, false) + c.Check(run.UsableAtRunTime(), equals, true) + + // That's all for parsing tool definitions from the pkgsrc infrastructure. + // See Test_Tools__package_Makefile for a continuation. +} + +func (s *Suite) Test_Tools__package_Makefile(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.CreateFileLines("mk/tools/defaults.mk", + "TOOLS_CREATE+= load", + "TOOLS_CREATE+= run", + "TOOLS_CREATE+= nowhere", + "TOOLS_CREATE+= pkg-before-prefs", + "TOOLS_CREATE+= pkg-after-prefs", + "_TOOLS_VARNAME.load= LOAD", + "_TOOLS_VARNAME.run= RUN_CMD", + "_TOOLS_VARNAME.nowhere= NOWHERE", + "_TOOLS_VARNAME.pkg-before-prefs= PKG_BEFORE_PREFS", + "_TOOLS_VARNAME.pkg-after-prefs= PKG_AFTER_PREFS") + t.CreateFileLines("mk/bsd.prefs.mk", + "USE_TOOLS+= load") + t.CreateFileLines("mk/bsd.pkg.mk", + "USE_TOOLS+= run") + G.Pkgsrc.LoadInfrastructure() + + tools := NewTools("") + tools.AddAll(G.Pkgsrc.Tools) + + load := tools.ByName("load") + run := tools.ByName("run") + nowhere := tools.ByName("nowhere") + before := tools.ByName("pkg-before-prefs") + after := tools.ByName("pkg-after-prefs") + + c.Check(load.UsableAtRunTime(), equals, true) + c.Check(run.UsableAtRunTime(), equals, true) + c.Check(nowhere.UsableAtRunTime(), equals, false) + + // The seenPrefs variable is only relevant for the package Makefile. + // All other files must not use the tools at load time. + // For them, seenPrefs can be though of as being true from the beginning. + + tools.ParseToolLine(t.NewMkLine("Makefile", 2, "USE_TOOLS+= pkg-before-prefs")) + + c.Check(before.Validity, equals, AfterPrefsMk) + c.Check(before.UsableAtLoadTime(false), equals, false) + c.Check(before.UsableAtLoadTime(true), equals, true) + c.Check(before.UsableAtRunTime(), equals, true) + + c.Check(tools.SeenPrefs, equals, false) + + tools.ParseToolLine(t.NewMkLine("Makefile", 3, ".include \"../../mk/bsd.prefs.mk\"")) + + c.Check(tools.SeenPrefs, equals, true) + + tools.ParseToolLine(t.NewMkLine("Makefile", 4, "USE_TOOLS+= pkg-after-prefs")) + + c.Check(after.Validity, equals, AtRunTime) + c.Check(after.UsableAtLoadTime(false), equals, false) + c.Check(after.UsableAtLoadTime(true), equals, false) + c.Check(after.UsableAtRunTime(), equals, true) +} + +func (s *Suite) Test_Tools__builtin_mk(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.SetupCommandLine("-Wall,no-space") + t.CreateFileLines("mk/tools/defaults.mk", + "TOOLS_CREATE+= load", + "TOOLS_CREATE+= run", + "TOOLS_CREATE+= nowhere", + "_TOOLS_VARNAME.load= LOAD", + "_TOOLS_VARNAME.run= RUN_CMD", + "_TOOLS_VARNAME.nowhere= NOWHERE") + t.CreateFileLines("mk/bsd.prefs.mk", + "USE_TOOLS+= load") + t.CreateFileLines("mk/bsd.pkg.mk", + "USE_TOOLS+= run") + t.CreateFileLines("mk/buildlink3/bsd.builtin.mk") + G.Pkgsrc.LoadInfrastructure() + + // Tools that are defined by pkgsrc as load-time tools + // may be used in any file at load time. + + mklines := t.NewMkLines("builtin.mk", + MkRcsID, + "", + "VAR!= ${ECHO} 'too early'", + "VAR!= ${LOAD} 'too early'", + "VAR!= ${RUN_CMD} 'never allowed'", + "VAR!= ${NOWHERE} 'never allowed'", + "", + ".include \"../../mk/buildlink3/bsd.builtin.mk\"", + "", + "VAR!= ${ECHO} 'valid'", + "VAR!= ${LOAD} 'valid'", + "VAR!= ${RUN_CMD} 'never allowed'", + "VAR!= ${NOWHERE} 'never allowed'", + "", + "VAR!= ${VAR}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: builtin.mk:3: To use the tool ${ECHO} at load time, bsd.prefs.mk has to be included before.", + "WARN: builtin.mk:4: To use the tool ${LOAD} at load time, bsd.prefs.mk has to be included before.", + "WARN: builtin.mk:5: The tool ${RUN_CMD} cannot be used at load time.", + "WARN: builtin.mk:6: The tool ${NOWHERE} cannot be used at load time.", + "WARN: builtin.mk:12: The tool ${RUN_CMD} cannot be used at load time.", + "WARN: builtin.mk:13: The tool ${NOWHERE} cannot be used at load time.") +} + +func (s *Suite) Test_Tools__implicit_definition_in_bsd_pkg_mk(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.SetupCommandLine("-Wall,no-space") + t.CreateFileLines("mk/tools/defaults.mk", + MkRcsID) // None + t.CreateFileLines("mk/bsd.prefs.mk", + "USE_TOOLS+= load") + t.CreateFileLines("mk/bsd.pkg.mk", + "USE_TOOLS+= run") + + // It's practically impossible that a tool is added to USE_TOOLS in + // bsd.pkg.mk and not defined earlier in mk/tools/defaults.mk, but + // the pkglint code is even prepared for these rare cases. + // In other words, this test is only there for the code coverage. + G.Pkgsrc.LoadInfrastructure() + + c.Check(G.Pkgsrc.Tools.ByName("run"), deepEquals, &Tool{"run", "", false, AtRunTime}) +} + +func (s *Suite) Test_Tools__both_prefs_and_pkg_mk(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.SetupCommandLine("-Wall,no-space") + t.CreateFileLines("mk/tools/defaults.mk", + MkRcsID) + t.CreateFileLines("mk/bsd.prefs.mk", + "USE_TOOLS+= both") + t.CreateFileLines("mk/bsd.pkg.mk", + "USE_TOOLS+= both") + + // The echo tool is mentioned in both files. The file bsd.prefs.mk + // grants more use cases (load time + run time), therefore it wins. + G.Pkgsrc.LoadInfrastructure() + + c.Check(G.Pkgsrc.Tools.ByName("both").Validity, equals, AfterPrefsMk) +} + +func (s *Suite) Test_Tools__tools_having_the_same_variable_name(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.SetupCommandLine("-Wall,no-space") + t.CreateFileLines("mk/tools/defaults.mk", + "_TOOLS_VARNAME.awk= AWK", + "_TOOLS_VARNAME.gawk= AWK", + "_TOOLS_VARNAME.gsed= SED", + "_TOOLS_VARNAME.sed= SED") + t.CreateFileLines("mk/bsd.prefs.mk", + "USE_TOOLS+= awk sed") + + G.Pkgsrc.LoadInfrastructure() + + c.Check(G.Pkgsrc.Tools.ByName("awk").Validity, equals, AfterPrefsMk) + c.Check(G.Pkgsrc.Tools.ByName("sed").Validity, equals, AfterPrefsMk) + c.Check(G.Pkgsrc.Tools.ByName("gawk").Validity, equals, Nowhere) + c.Check(G.Pkgsrc.Tools.ByName("gsed").Validity, equals, Nowhere) + + t.EnableTracingToLog() + G.Pkgsrc.Tools.Trace() + t.DisableTracing() + + t.CheckOutputLines( + "TRACE: + (*Tools).Trace(\"Pkgsrc\")", + "TRACE: 1 tool &{Name:awk Varname:AWK MustUseVarForm:false Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:echo Varname:ECHO MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:echo -n Varname:ECHO_N MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:false Varname:FALSE MustUseVarForm:true Validity:Nowhere}", + "TRACE: 1 tool &{Name:gawk Varname:AWK MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:gsed Varname:SED MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:sed Varname:SED MustUseVarForm:false Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:test Varname:TEST MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:true Varname:TRUE MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: - (*Tools).Trace(\"Pkgsrc\")") + + tools := NewTools("module.mk") + tools.AddAll(G.Pkgsrc.Tools) + + t.EnableTracingToLog() + tools.Trace() + t.DisableTracing() + + t.CheckOutputLines( + "TRACE: + (*Tools).Trace(\"module.mk\")", + "TRACE: 1 tool &{Name:awk Varname:AWK MustUseVarForm:false Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:echo Varname:ECHO MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:echo -n Varname:ECHO_N MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:false Varname:FALSE MustUseVarForm:true Validity:Nowhere}", + "TRACE: 1 tool &{Name:gawk Varname:AWK MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:gsed Varname:SED MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:sed Varname:SED MustUseVarForm:false Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:test Varname:TEST MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:true Varname:TRUE MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: - (*Tools).Trace(\"module.mk\")") +} + +func (s *Suite) Test_ToolTime_String(c *check.C) { + c.Check(LoadTime.String(), equals, "LoadTime") + c.Check(RunTime.String(), equals, "RunTime") +} + +func (s *Suite) Test_Tools__var(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + t.SetupPkgsrc() + t.CreateFileLines("mk/tools/defaults.mk", + "TOOLS_CREATE+= ln", + "_TOOLS_VARNAME.ln= LN") + t.CreateFileLines("mk/bsd.pkg.mk", + "USE_TOOLS+= ln") + G.Pkgsrc.LoadInfrastructure() + + mklines := t.NewMkLines("module.mk", + MkRcsID, + "", + "pre-configure:", + "\t${LN} from to") + + mklines.Check() + + t.CheckOutputEmpty() +} + +// Demonstrates how the Tools type handles tool that share the same +// variable name. Of these tools, the GNU variant is preferred. +// +// See also Pkglint.Tool. +func (s *Suite) Test_Tools_AddAll__tools_having_the_same_variable_name(c *check.C) { + nonGnu := NewTools("non-gnu") + nonGnu.Define("sed", "SED", dummyMkLine).SetValidity(AfterPrefsMk, "") + + gnu := NewTools("gnu") + gnu.Define("gsed", "SED", dummyMkLine) + + local1 := NewTools("local") + local1.AddAll(nonGnu) + local1.AddAll(gnu) + + c.Check(local1.ByName("sed").Validity, equals, AfterPrefsMk) + c.Check(local1.ByName("gsed").Validity, equals, Nowhere) + local1.Trace() + + local2 := NewTools("local") + local2.AddAll(gnu) + local2.AddAll(nonGnu) + + c.Check(local2.ByName("sed").Validity, equals, AfterPrefsMk) + c.Check(local2.ByName("gsed").Validity, equals, Nowhere) + local2.Trace() + + nonGnu.ByName("sed").Validity = Nowhere + gnu.ByName("gsed").Validity = AfterPrefsMk + + local3 := NewTools("local") + local3.AddAll(nonGnu) + local3.AddAll(gnu) + + c.Check(local3.ByName("sed").Validity, equals, Nowhere) + c.Check(local3.ByName("gsed").Validity, equals, AfterPrefsMk) + local3.Trace() + + local4 := NewTools("local") + local4.AddAll(gnu) + local4.AddAll(nonGnu) + + c.Check(local4.ByName("sed").Validity, equals, Nowhere) + c.Check(local4.ByName("gsed").Validity, equals, AfterPrefsMk) + local4.Trace() + + c.Check(local1, deepEquals, local2) + c.Check(local4, deepEquals, local4) } Index: pkgsrc/pkgtools/pkglint/files/autofix.go diff -u pkgsrc/pkgtools/pkglint/files/autofix.go:1.8 pkgsrc/pkgtools/pkglint/files/autofix.go:1.9 --- pkgsrc/pkgtools/pkglint/files/autofix.go:1.8 Thu Jul 12 16:23:36 2018 +++ pkgsrc/pkgtools/pkglint/files/autofix.go Wed Sep 5 17:56:22 2018 @@ -139,7 +139,7 @@ func (fix *Autofix) Realign(mkline MkLin if (oldWidth == 0 || width < oldWidth) && width >= 8 && rawLine.textnl != "\n" { oldWidth = width } - if !regex.Matches(space, `^\t* {0,7}`) { + if !regex.Matches(space, `^\t* {0,7}$`) { normalized = false } } @@ -346,12 +346,14 @@ func SaveAutofixChanges(lines []Line) (a } err := ioutil.WriteFile(tmpname, []byte(text), 0666) if err != nil { - NewLineWhole(tmpname).Errorf("Cannot write.") + logs(llError, tmpname, "", "Cannot write: %s", "Cannot write: "+err.Error()) continue } err = os.Rename(tmpname, fname) if err != nil { - NewLineWhole(fname).Errorf("Cannot overwrite with auto-fixed content.") + logs(llError, tmpname, "", + "Cannot overwrite with auto-fixed content: %s", + "Cannot overwrite with auto-fixed content: "+err.Error()) continue } autofixed = true Index: pkgsrc/pkgtools/pkglint/files/parser_test.go diff -u pkgsrc/pkgtools/pkglint/files/parser_test.go:1.8 pkgsrc/pkgtools/pkglint/files/parser_test.go:1.9 --- pkgsrc/pkgtools/pkglint/files/parser_test.go:1.8 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/parser_test.go Wed Sep 5 17:56:22 2018 @@ -1,7 +1,7 @@ package main import ( - check "gopkg.in/check.v1" + "gopkg.in/check.v1" ) func (s *Suite) Test_Parser_PkgbasePattern(c *check.C) { Index: pkgsrc/pkgtools/pkglint/files/pkgsrc.go diff -u pkgsrc/pkgtools/pkglint/files/pkgsrc.go:1.8 pkgsrc/pkgtools/pkglint/files/pkgsrc.go:1.9 --- pkgsrc/pkgtools/pkglint/files/pkgsrc.go:1.8 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/pkgsrc.go Wed Sep 5 17:56:22 2018 @@ -20,7 +20,7 @@ type Pkgsrc struct { // within the bsd.pkg.mk file. buildDefs map[string]bool - Tools ToolRegistry + Tools Tools MasterSiteURLToVar map[string]string // "https://github.com/" => "MASTER_SITE_GITHUB" MasterSiteVarToURL map[string]string // "MASTER_SITE_GITHUB" => "https://github.com/" @@ -32,7 +32,7 @@ type Pkgsrc struct { LastChange map[string]*Change // latest map[string]string // "lang/php[0-9]*" => "lang/php70" - UserDefinedVars map[string]MkLine // varname => line; used for checking BUILD_DEFS + UserDefinedVars Scope // Used for checking BUILD_DEFS Deprecated map[string]string // vartypes map[string]*Vartype // varcanon => type @@ -44,7 +44,7 @@ func NewPkgsrc(dir string) *Pkgsrc { src := &Pkgsrc{ dir, make(map[string]bool), - NewToolRegistry(), + NewTools("Pkgsrc"), make(map[string]string), make(map[string]string), make(map[string]string), @@ -52,7 +52,7 @@ func NewPkgsrc(dir string) *Pkgsrc { nil, make(map[string]*Change), make(map[string]string), - make(map[string]MkLine), + NewScope(), make(map[string]string), make(map[string]*Vartype), nil, // Only initialized when pkglint is run for a whole pkgsrc installation @@ -61,20 +61,53 @@ func NewPkgsrc(dir string) *Pkgsrc { // Some user-defined variables do not influence the binary // package at all and therefore do not have to be added to // BUILD_DEFS; therefore they are marked as "already added". - src.AddBuildDef("DISTDIR") - src.AddBuildDef("FETCH_CMD") - src.AddBuildDef("FETCH_OUTPUT_ARGS") - - // The following variables are not expected to be modified - // by the pkgsrc user. They are added here to prevent unnecessary - // warnings by pkglint. - src.AddBuildDef("GAMES_USER") - src.AddBuildDef("GAMES_GROUP") - src.AddBuildDef("GAMEDATAMODE") - src.AddBuildDef("GAMEDIRMODE") - src.AddBuildDef("GAMEMODE") - src.AddBuildDef("GAMEOWN") - src.AddBuildDef("GAMEGRP") + src.AddBuildDefs("DISTDIR", "FETCH_CMD", "FETCH_OUTPUT_ARGS") + + // The following variables are added to _BUILD_DEFS by the pkgsrc + // infrastructure and thus don't need to be added by the package again. + // To regenerate the below list: + // grep -hr '^_BUILD_DEFS+=' mk/ | tr ' \t' '\n\n' | sed -e 's,.*=,,' -e '/^_/d' -e '/^$/d' -e 's,.*,"&"\,,' | sort -u + src.AddBuildDefs("PKG_HACKS") + src.AddBuildDefs( + "ABI", + "BUILTIN_PKGS", + "CFLAGS", + "CMAKE_ARGS", + "CONFIGURE_ARGS", + "CONFIGURE_ENV", + "CPPFLAGS", + "FFLAGS", + "GAMEDATAMODE", + "GAMEDIRMODE", + "GAMEMODE", + "GAMES_GROUP", + "GAMES_USER", + "GLIBC_VERSION", + "INIT_SYSTEM", + "LDFLAGS", + "LICENSE", + "LOCALBASE", + "MACHINE_ARCH", + "MACHINE_GNU_ARCH", + "MULTI", + "NO_BIN_ON_CDROM", + "NO_BIN_ON_FTP", + "NO_SRC_ON_CDROM", + "NO_SRC_ON_FTP", + "OBJECT_FMT", + "OPSYS", + "OS_VERSION", + "OSVERSION_SPECIFIC", + "PKG_HACKS", + "PKG_OPTIONS", + "PKG_SYSCONFBASEDIR", + "PKG_SYSCONFDIR", + "PKGGNUDIR", + "PKGINFODIR", + "PKGMANDIR", + "PKGPATH", + "RESTRICTED", + "USE_ABI_DEPENDS") return src } @@ -110,10 +143,6 @@ func (src *Pkgsrc) Latest(category strin return latest } - if src.latest == nil { - src.latest = make(map[string]string) - } - categoryDir := src.File(category) error := func() string { dummyLine.Errorf("Cannot find latest version of %q in %q.", re, categoryDir) @@ -145,6 +174,8 @@ func (src *Pkgsrc) Latest(category strin // loadTools loads the tool definitions from `mk/tools/*`. func (src *Pkgsrc) loadTools() { + tools := src.Tools + toolFiles := []string{"defaults.mk"} { toc := G.Pkgsrc.File("mk/tools/bsd.tools.mk") @@ -162,63 +193,57 @@ func (src *Pkgsrc) loadTools() { } } - reg := src.Tools - reg.RegisterTool(&Tool{"echo", "ECHO", true, true, true}, dummyMkLine) - reg.RegisterTool(&Tool{"echo -n", "ECHO_N", true, true, true}, dummyMkLine) - reg.RegisterTool(&Tool{"false", "FALSE", true /*why?*/, true, false}, dummyMkLine) - reg.RegisterTool(&Tool{"test", "TEST", true, true, true}, dummyMkLine) - reg.RegisterTool(&Tool{"true", "TRUE", true /*why?*/, true, true}, dummyMkLine) + // TODO: parse bsd.prefs.mk instead of hardcoding this. + toolDefs := []struct { + Name string + Varname string + }{ + {"echo", "ECHO"}, + {"echo -n", "ECHO_N"}, + {"false", "FALSE"}, + {"test", "TEST"}, + {"true", "TRUE"}} + + for _, toolDef := range toolDefs { + tool := tools.Define(toolDef.Name, toolDef.Varname, dummyMkLine) + tool.MustUseVarForm = true + if toolDef.Name != "false" { + tool.SetValidity(AfterPrefsMk, tools.TraceName) + } + } for _, basename := range toolFiles { mklines := G.Pkgsrc.LoadMk("mk/tools/"+basename, MustSucceed|NotEmpty) for _, mkline := range mklines.mklines { - reg.ParseToolLine(mkline) + tools.ParseToolLineCreate(mkline, true) } } for _, relativeName := range [...]string{"mk/bsd.prefs.mk", "mk/bsd.pkg.mk"} { - dirDepth := 0 mklines := G.Pkgsrc.LoadMk(relativeName, MustSucceed|NotEmpty) for _, mkline := range mklines.mklines { if mkline.IsVarassign() { - varname := mkline.Varname() - value := mkline.Value() - if varname == "USE_TOOLS" { - if trace.Tracing { - trace.Stepf("[dirDepth=%d] %s", dirDepth, value) - } - if dirDepth == 0 || dirDepth == 1 && relativeName == "mk/bsd.prefs.mk" { - for _, toolname := range splitOnSpace(value) { - if !containsVarRef(toolname) { - tool := reg.Register(toolname, mkline) - tool.Predefined = true - if relativeName == "mk/bsd.prefs.mk" { - tool.UsableAtLoadTime = true - } - } - } + switch mkline.Varname() { + case "USE_TOOLS": + // Since this line is in the pkgsrc infrastructure, each tool mentioned + // in USE_TOOLS is trusted to be also defined somewhere in the actual + // list of available tools. + // + // This assumption does not work for processing USE_TOOLS in packages, though. + tools.ParseToolLineCreate(mkline, true) + + case "_BUILD_DEFS": + for _, bdvar := range mkline.ValueSplit(mkline.Value(), "") { + src.AddBuildDefs(bdvar) } - - } else if varname == "_BUILD_DEFS" { - for _, bdvar := range splitOnSpace(value) { - src.AddBuildDef(bdvar) - } - } - - } else if mkline.IsDirective() { - switch mkline.Directive() { - case "if", "ifdef", "ifndef", "for": - dirDepth++ - case "endif", "endfor": - dirDepth-- } } } } if trace.Tracing { - reg.Trace() + tools.Trace() } } @@ -243,10 +268,10 @@ func (src *Pkgsrc) parseSuggestedUpdates if m, pkgbase, pkgversion := match2(pkgname, rePkgname); m { updates = append(updates, SuggestedUpdate{line, pkgbase, pkgversion, comment}) } else { - line.Warnf("Invalid package name %q", pkgname) + line.Warnf("Invalid package name %q.", pkgname) } } else { - line.Warnf("Invalid line format %q", text) + line.Warnf("Invalid line format %q.", text) } } } @@ -366,7 +391,7 @@ func (src *Pkgsrc) loadUserDefinedVars() for _, mkline := range mklines.mklines { if mkline.IsVarassign() { - src.UserDefinedVars[mkline.Varname()] = mkline + src.UserDefinedVars.Define(mkline.Varname(), mkline) } } } @@ -503,7 +528,7 @@ func (src *Pkgsrc) initDeprecatedVars() "LICENCE": "Use LICENSE instead.", // November 2007 - //USE_NCURSES Include "../../devel/ncurses/buildlink3.mk" instead. + //USE_NCURSES: Include "../../devel/ncurses/buildlink3.mk" instead. // December 2007 "INSTALLATION_DIRS_FROM_PLIST": "Use AUTO_MKDIRS instead.", @@ -558,8 +583,10 @@ func (src *Pkgsrc) ToRel(fileName string return relpath(src.topdir, fileName) } -func (src *Pkgsrc) AddBuildDef(varname string) { - src.buildDefs[varname] = true +func (src *Pkgsrc) AddBuildDefs(varnames ...string) { + for _, varname := range varnames { + src.buildDefs[varname] = true + } } func (src *Pkgsrc) IsBuildDef(varname string) bool { @@ -607,6 +634,85 @@ func (src *Pkgsrc) loadPkgOptions() { } } +// VariableType returns the type of the variable +// (possibly guessed based on the variable name), +// or nil if the type cannot even be guessed. +func (src *Pkgsrc) VariableType(varname string) (vartype *Vartype) { + if trace.Tracing { + defer trace.Call(varname, trace.Result(&vartype))() + } + + if vartype := src.vartypes[varname]; vartype != nil { + return vartype + } + if vartype := src.vartypes[varnameCanon(varname)]; vartype != nil { + return vartype + } + + if tool := G.ToolByVarname(varname, RunTime); tool != nil { + if trace.Tracing { + trace.Stepf("Use of tool %+v", tool) + } + perms := aclpUse + if tool.Validity == AfterPrefsMk && G.Mk.Tools.SeenPrefs { + perms |= aclpUseLoadtime + } + return &Vartype{lkNone, BtShellCommand, []ACLEntry{{"*", perms}}, false} + } + + if m, toolVarname := match1(varname, `^TOOLS_(.*)`); m { + if tool := G.ToolByVarname(toolVarname, RunTime); tool != nil { + return &Vartype{lkNone, BtPathname, []ACLEntry{{"*", aclpUse}}, false} + } + } + + allowAll := []ACLEntry{{"*", aclpAll}} + allowRuntime := []ACLEntry{{"*", aclpAllRuntime}} + + // Guess the data type of the variable based on naming conventions. + varbase := varnameBase(varname) + var gtype *Vartype + switch { + case hasSuffix(varbase, "DIRS"): + gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} + case hasSuffix(varbase, "DIR") && !hasSuffix(varbase, "DESTDIR"), hasSuffix(varname, "_HOME"): + gtype = &Vartype{lkNone, BtPathname, allowRuntime, true} + case hasSuffix(varbase, "FILES"): + gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} + case hasSuffix(varbase, "FILE"): + gtype = &Vartype{lkNone, BtPathname, allowRuntime, true} + case hasSuffix(varbase, "PATH"): + gtype = &Vartype{lkNone, BtPathlist, allowRuntime, true} + case hasSuffix(varbase, "PATHS"): + gtype = &Vartype{lkShell, BtPathname, allowRuntime, true} + case hasSuffix(varbase, "_USER"): + gtype = &Vartype{lkNone, BtUserGroupName, allowAll, true} + case hasSuffix(varbase, "_GROUP"): + gtype = &Vartype{lkNone, BtUserGroupName, allowAll, true} + case hasSuffix(varbase, "_ENV"): + gtype = &Vartype{lkShell, BtShellWord, allowRuntime, true} + case hasSuffix(varbase, "_CMD"): + gtype = &Vartype{lkNone, BtShellCommand, allowRuntime, true} + case hasSuffix(varbase, "_ARGS"): + gtype = &Vartype{lkShell, BtShellWord, allowRuntime, true} + case hasSuffix(varbase, "_CFLAGS"), hasSuffix(varname, "_CPPFLAGS"), hasSuffix(varname, "_CXXFLAGS"): + gtype = &Vartype{lkShell, BtCFlag, allowRuntime, true} + case hasSuffix(varname, "_LDFLAGS"): + gtype = &Vartype{lkShell, BtLdFlag, allowRuntime, true} + case hasSuffix(varbase, "_MK"): + gtype = &Vartype{lkNone, BtUnknown, allowAll, true} + } + + if trace.Tracing { + if gtype != nil { + trace.Step2("The guessed type of %q is %q.", varname, gtype.String()) + } else { + trace.Step1("No type definition found for %q.", varname) + } + } + return gtype +} + // Change is a change entry from the `doc/CHANGES-*` files. type Change struct { Line Line Index: pkgsrc/pkgtools/pkglint/files/shtypes.go diff -u pkgsrc/pkgtools/pkglint/files/shtypes.go:1.8 pkgsrc/pkgtools/pkglint/files/shtypes.go:1.9 --- pkgsrc/pkgtools/pkglint/files/shtypes.go:1.8 Sat Apr 28 23:32:52 2018 +++ pkgsrc/pkgtools/pkglint/files/shtypes.go Wed Sep 5 17:56:22 2018 @@ -39,7 +39,7 @@ func (t ShAtomType) IsWord() bool { type ShAtom struct { Type ShAtomType MkText string - Quoting ShQuoting + Quoting ShQuoting // The quoting state at the end of the token Data interface{} } Index: pkgsrc/pkgtools/pkglint/files/autofix_test.go diff -u pkgsrc/pkgtools/pkglint/files/autofix_test.go:1.9 pkgsrc/pkgtools/pkglint/files/autofix_test.go:1.10 --- pkgsrc/pkgtools/pkglint/files/autofix_test.go:1.9 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/autofix_test.go Wed Sep 5 17:56:22 2018 @@ -2,6 +2,8 @@ package main import ( "gopkg.in/check.v1" + "os" + "runtime" "strings" ) @@ -469,6 +471,8 @@ func (s *Suite) Test_Autofix__skip(c *ch fix.InsertBefore("before") fix.InsertAfter("after") fix.Delete() + fix.Custom(func(printAutofix, autofix bool) {}) + fix.Realign(dummyMkLine, 32) fix.Apply() SaveAutofixChanges(lines) @@ -478,3 +482,101 @@ func (s *Suite) Test_Autofix__skip(c *ch "111 222 333 444 555") c.Check(lines[0].raw[0].textnl, equals, "111 222 333 444 555\n") } + +func (s *Suite) Test_Autofix_Apply__panic(c *check.C) { + t := s.Init(c) + + line := t.NewLine("filename", 123, "text") + + c.Assert(func() { + fix := line.Autofix() + fix.Apply() + }, check.Panics, "Each autofix must have a diagnostic.") + + c.Assert(func() { + fix := line.Autofix() + fix.Replace("from", "to") + fix.Apply() + }, check.Panics, "Autofix: The diagnostic must be given before the action.") + + c.Assert(func() { + fix := line.Autofix() + fix.Warnf("Warning without period") + fix.Apply() + }, check.Panics, "Autofix: format \"Warning without period\" must end with a period.") +} + +func (s *Suite) Test_Autofix_Apply__file_removed(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("--autofix") + lines := t.SetupFileLines("subdir/file.txt", + "line 1") + os.RemoveAll(t.File("subdir")) + + fix := lines[0].Autofix() + fix.Warnf("Should start with an uppercase letter.") + fix.Replace("line", "Line") + fix.Apply() + + SaveAutofixChanges(lines) + + c.Check(t.Output(), check.Matches, ""+ + "AUTOFIX: ~/subdir/file.txt:1: Replacing \"line\" with \"Line\".\n"+ + "ERROR: ~/subdir/file.txt.pkglint.tmp: Cannot write: .*\n") +} + +func (s *Suite) Test_Autofix_Apply__file_busy_Windows(c *check.C) { + t := s.Init(c) + + if runtime.GOOS != "windows" { + return + } + + t.SetupCommandLine("--autofix") + lines := t.SetupFileLines("subdir/file.txt", + "line 1") + + // As long as the file is kept open, it cannot be overwritten or deleted. + openFile, err := os.OpenFile(t.File("subdir/file.txt"), 0, 0666) + defer openFile.Close() + c.Check(err, check.IsNil) + + fix := lines[0].Autofix() + fix.Warnf("Should start with an uppercase letter.") + fix.Replace("line", "Line") + fix.Apply() + + SaveAutofixChanges(lines) + + c.Check(t.Output(), check.Matches, ""+ + "AUTOFIX: ~/subdir/file.txt:1: Replacing \"line\" with \"Line\".\n"+ + "ERROR: ~/subdir/file.txt.pkglint.tmp: Cannot overwrite with auto-fixed content: .*\n") +} + +// This test tests the highly unlikely situation in which a file is loaded +// by pkglint, and just before writing the autofixed content back, another +// process takes the file and replaces it with a directory of the same name. +// +// 100% code coverage sometimes requires creativity. :) +func (s *Suite) Test_Autofix_Apply__file_converted_to_directory(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("--autofix") + lines := t.SetupFileLines("file.txt", + "line 1") + + c.Check(os.RemoveAll(t.File("file.txt")), check.IsNil) + c.Check(os.MkdirAll(t.File("file.txt"), 0777), check.IsNil) + + fix := lines[0].Autofix() + fix.Warnf("Should start with an uppercase letter.") + fix.Replace("line", "Line") + fix.Apply() + + SaveAutofixChanges(lines) + + c.Check(t.Output(), check.Matches, ""+ + "AUTOFIX: ~/file.txt:1: Replacing \"line\" with \"Line\".\n"+ + "ERROR: ~/file.txt.pkglint.tmp: Cannot overwrite with auto-fixed content: .*\n") +} Index: pkgsrc/pkgtools/pkglint/files/shtokenizer.go diff -u pkgsrc/pkgtools/pkglint/files/shtokenizer.go:1.9 pkgsrc/pkgtools/pkglint/files/shtokenizer.go:1.10 --- pkgsrc/pkgtools/pkglint/files/shtokenizer.go:1.9 Thu Jul 12 16:23:36 2018 +++ pkgsrc/pkgtools/pkglint/files/shtokenizer.go Wed Sep 5 17:56:22 2018 @@ -1,9 +1,5 @@ package main -import ( - "netbsd.org/pkglint/textproc" -) - type ShTokenizer struct { parser *Parser mkp *MkParser @@ -15,6 +11,9 @@ func NewShTokenizer(line Line, text stri return &ShTokenizer{p, mkp} } +// ShAtom parses a basic building block of a shell program. +// Examples for such atoms are: variable reference, operator, text, quote, space. +// // See ShQuote.Feed func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { if p.parser.EOF() { @@ -84,10 +83,9 @@ func (p *ShTokenizer) shAtomPlain() *ShA return &ShAtom{shtComment, repl.Group(0), q, nil} case repl.AdvanceStr("$$("): return &ShAtom{shtSubshell, repl.Str(), q, nil} - case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]^_a-z{}~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.Group(0), q, nil} } - return nil + + return p.shAtomInternal(q, false, false) } func (p *ShTokenizer) shAtomDquot() *ShAtom { @@ -97,10 +95,8 @@ func (p *ShTokenizer) shAtomDquot() *ShA return &ShAtom{shtWord, repl.Str(), shqPlain, nil} case repl.AdvanceStr("`"): return &ShAtom{shtWord, repl.Str(), shqDquotBackt, nil} - case repl.AdvanceRegexp(`^(?:[\t !#%&'()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.Group(0), shqDquot, nil} // XXX: unescape? } - return nil + return p.shAtomInternal(shqDquot, true, false) } func (p *ShTokenizer) shAtomSquot() *ShAtom { @@ -108,10 +104,8 @@ func (p *ShTokenizer) shAtomSquot() *ShA switch { case repl.AdvanceStr("'"): return &ShAtom{shtWord, repl.Str(), shqPlain, nil} - case repl.AdvanceRegexp(`^([\t !"#%&()*+,\-./0-9:;<=>?@A-Z\[\\\]^_` + "`" + `a-z{|}~]+|\$\$)+`): - return &ShAtom{shtWord, repl.Group(0), shqSquot, nil} } - return nil + return p.shAtomInternal(shqSquot, false, true) } func (p *ShTokenizer) shAtomBackt() *ShAtom { @@ -131,10 +125,8 @@ func (p *ShTokenizer) shAtomBackt() *ShA return &ShAtom{shtSpace, repl.Str(), q, nil} case repl.AdvanceRegexp("^#[^`]*"): return &ShAtom{shtComment, repl.Str(), q, nil} - case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]_a-z~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.Str(), q, nil} } - return nil + return p.shAtomInternal(q, false, false) } // In pkgsrc, the $(...) subshell syntax is not used to preserve @@ -249,6 +241,47 @@ func (p *ShTokenizer) shAtomDquotBacktSq return nil } +// shAtomInternal advances the parser over the next "word", +// which is everything that does not change the quoting and is not a Make(1) variable. +// Shell variables may appear as part of a word. +// +// Examples: +// while$var +// $$, +// $$!$$$$ +// echo +// text${var:=default}text +func (p *ShTokenizer) shAtomInternal(q ShQuoting, dquot, squot bool) *ShAtom { + repl := p.parser.repl + + mark := repl.Mark() +loop: + for { + _ = `^[\t "$&'();<>\\|]+` // These are not allowed in shqPlain. + + switch { + case repl.AdvanceRegexp(`^[!#%*+,\-./0-9:=?@A-Z\[\]^_a-z{}~]+`): + case dquot && repl.AdvanceRegexp(`^[\t &'();<>|]+`): + case squot && repl.AdvanceByte('`'): + case squot && repl.AdvanceRegexp(`^[\t "&();<>\\|]+`): + case squot && repl.AdvanceStr("$$"): + case squot: + break loop + case repl.AdvanceRegexp(`^\\[^$]`): + case repl.HasPrefixRegexp(`^\$\$[^!#(*\-0-9?@A-Z_a-z{]`): + repl.AdvanceStr("$$") + case repl.AdvanceRegexp(`^(?:` + reShDollar + `)`): + default: + break loop + } + } + + if token := repl.Since(mark); token != "" { + return &ShAtom{shtWord, token, q, nil} + } + return nil +} + func (p *ShTokenizer) shOperator(q ShQuoting) *ShAtom { repl := p.parser.repl switch { @@ -298,12 +331,12 @@ func (p *ShTokenizer) ShToken() *ShToken } repl := p.parser.repl - inimark := repl.Mark() + initialMark := repl.Mark() var atoms []*ShAtom for peek() != nil && peek().Type == shtSpace { skip() - inimark = repl.Mark() + initialMark = repl.Mark() } if peek() == nil { @@ -313,28 +346,20 @@ func (p *ShTokenizer) ShToken() *ShToken return NewShToken(atom.MkText, atom) } -nextatom: +nextAtom: mark := repl.Mark() atom := peek() if atom != nil && (atom.Type.IsWord() || atom.Quoting != shqPlain) { skip() atoms = append(atoms, atom) - goto nextatom + goto nextAtom } repl.Reset(mark) if len(atoms) == 0 { return nil } - return NewShToken(repl.Since(inimark), atoms...) -} - -func (p *ShTokenizer) Mark() textproc.PrefixReplacerMark { - return p.parser.repl.Mark() -} - -func (p *ShTokenizer) Reset(mark textproc.PrefixReplacerMark) { - p.parser.repl.Reset(mark) + return NewShToken(repl.Since(initialMark), atoms...) } func (p *ShTokenizer) Rest() string { Index: pkgsrc/pkgtools/pkglint/files/buildlink3_test.go diff -u pkgsrc/pkgtools/pkglint/files/buildlink3_test.go:1.16 pkgsrc/pkgtools/pkglint/files/buildlink3_test.go:1.17 --- pkgsrc/pkgtools/pkglint/files/buildlink3_test.go:1.16 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/buildlink3_test.go Wed Sep 5 17:56:22 2018 @@ -102,6 +102,7 @@ func (s *Suite) Test_ChecklinesBuildlink "", "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11>=1.6.1", "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X12>=1.6.1.2nb2", + "BUILDLINK_ABI_DEPENDS.hs-X12+=\ths-X11>=1.6.1.2nb2", "", ".endif\t# HS_X11_BUILDLINK3_MK", "", @@ -110,7 +111,8 @@ func (s *Suite) Test_ChecklinesBuildlink ChecklinesBuildlink3Mk(mklines) t.CheckOutputLines( - "WARN: buildlink3.mk:9: Package name mismatch between ABI \"hs-X12\" and API \"hs-X11\" (from line 8).") + "WARN: buildlink3.mk:9: Package name mismatch between ABI \"hs-X12\" and API \"hs-X11\" (from line 8).", + "WARN: buildlink3.mk:10: Only buildlink variables for \"hs-X11\", not \"hs-X12\" may be set in this file.") } func (s *Suite) Test_ChecklinesBuildlink3Mk_abi_api_versions(c *check.C) { @@ -345,3 +347,43 @@ func (s *Suite) Test_ChecklinesBuildlink "ERROR: ~/buildlink3.mk:13: \"x11/libX11/buildlink3.mk\" does not exist.", "WARN: ~/buildlink3.mk:3: Expected a BUILDLINK_TREE line.") } + +func (s *Suite) Test_ChecklinesBuildlink3Mk__coverage(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + t.CreateFileLines("mk/pkg-build-options.mk") + t.CreateFileLines("category/dependency/buildlink3.mk") + mklines := t.SetupFileMkLines("category/package/buildlink3.mk", + MkRcsID, + "", + "BUILDLINK_TREE+=\ths-X11", + "", + ".if !defined(HS_X11_BUILDLINK3_MK)", + "HS_X11_BUILDLINK3_MK:=", + "", + "pkgbase := dependency", + ".include \"../../mk/pkg-build-options.mk\"", + "", + "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11>=1.6.1", + "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X11>=1.6.1.2nb2", + "", + ".include \"../../category/dependency/buildlink3.mk\"", + "", + ".if ${OPSYS} == \"NetBSD\"", + ".endif", + "", + ".for var in value", + ".endfor", + "", + ".endif\t# HS_X11_BUILDLINK3_MK", + "", + "BUILDLINK_TREE+=\t-hs-X11", + "", + "# the end") + + ChecklinesBuildlink3Mk(mklines) + + t.CheckOutputLines( + "WARN: ~/category/package/buildlink3.mk:25: The file should end here.") +} Index: pkgsrc/pkgtools/pkglint/files/distinfo_test.go diff -u pkgsrc/pkgtools/pkglint/files/distinfo_test.go:1.16 pkgsrc/pkgtools/pkglint/files/distinfo_test.go:1.17 --- pkgsrc/pkgtools/pkglint/files/distinfo_test.go:1.16 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/distinfo_test.go Wed Sep 5 17:56:22 2018 @@ -42,13 +42,15 @@ func (s *Suite) Test_ChecklinesDistinfo_ lines := t.NewLines("distinfo", RcsID, "", - "SHA512 (pkgname-1.0.tar.gz) = 12341234") + "SHA512 (pkgname-1.0.tar.gz) = 12341234", + "SHA512 (pkgname-1.1.tar.gz) = 12341234") ChecklinesDistinfo(lines) t.CheckOutputLines( "ERROR: distinfo:3: The hash SHA512 for pkgname-1.0.tar.gz is 12341234, which differs from Some-512-bit-hash in other/distinfo:7.", - "ERROR: distinfo:EOF: Expected SHA1, RMD160, SHA512, Size checksums for \"pkgname-1.0.tar.gz\", got SHA512.") + "ERROR: distinfo:4: Expected SHA1, RMD160, SHA512, Size checksums for \"pkgname-1.0.tar.gz\", got SHA512.", + "ERROR: distinfo:EOF: Expected SHA1, RMD160, SHA512, Size checksums for \"pkgname-1.1.tar.gz\", got SHA512.") } func (s *Suite) Test_ChecklinesDistinfo_uncommitted_patch(c *check.C) { @@ -184,3 +186,15 @@ func (s *Suite) Test_ChecklinesDistinfo_ t.CheckOutputEmpty() } + +func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1(c *check.C) { + t := s.Init(c) + + G.Pkg = NewPackage(t.File("category/package")) + + checker := &distinfoLinesChecker{} + checker.checkPatchSha1(dummyLine, "patch-nonexistent", "distinfo-sha1") + + t.CheckOutputLines( + "ERROR: patch-nonexistent does not exist.") +} Index: pkgsrc/pkgtools/pkglint/files/files_test.go diff -u pkgsrc/pkgtools/pkglint/files/files_test.go:1.16 pkgsrc/pkgtools/pkglint/files/files_test.go:1.17 --- pkgsrc/pkgtools/pkglint/files/files_test.go:1.16 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/files_test.go Wed Sep 5 17:56:22 2018 @@ -116,9 +116,9 @@ func (s *Suite) Test_convertToLogicalLin // This is just a side-effect and not relevant for this particular test. t.CheckOutputLines( - "ERROR: ~/comment.mk:15: Unknown Makefile line format.", - "ERROR: ~/comment.mk:19: Unknown Makefile line format.", - "ERROR: ~/comment.mk:23: Unknown Makefile line format.") + "ERROR: ~/comment.mk:15: Unknown Makefile line format: \"This is no comment\".", + "ERROR: ~/comment.mk:19: Unknown Makefile line format: \"This is no comment\".", + "ERROR: ~/comment.mk:23: Unknown Makefile line format: \"This is no comment\".") } func (s *Suite) Test_convertToLogicalLines_continuationInLastLine(c *check.C) { @@ -156,17 +156,11 @@ func (s *Suite) Test_Load(c *check.C) { t.CreateFileLines("empty") - func() { - defer t.ExpectFatalError() - Load(t.File("does-not-exist"), MustSucceed) - }() - - func() { - defer t.ExpectFatalError() - Load(t.File("empty"), MustSucceed|NotEmpty) - }() + t.ExpectFatal( + func() { Load(t.File("does-not-exist"), MustSucceed) }, + "FATAL: ~/does-not-exist: Cannot be read.") - t.CheckOutputLines( - "FATAL: ~/does-not-exist: Cannot be read.", + t.ExpectFatal( + func() { Load(t.File("empty"), MustSucceed|NotEmpty) }, "FATAL: ~/empty: Must not be empty.") } Index: pkgsrc/pkgtools/pkglint/files/category_test.go diff -u pkgsrc/pkgtools/pkglint/files/category_test.go:1.11 pkgsrc/pkgtools/pkglint/files/category_test.go:1.12 --- pkgsrc/pkgtools/pkglint/files/category_test.go:1.11 Sun Aug 12 16:31:56 2018 +++ pkgsrc/pkgtools/pkglint/files/category_test.go Wed Sep 5 17:56:22 2018 @@ -2,7 +2,7 @@ package main import "gopkg.in/check.v1" -func (s *Suite) Test_CheckdirCategory_totally_broken(c *check.C) { +func (s *Suite) Test_CheckdirCategory__totally_broken(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -32,7 +32,7 @@ func (s *Suite) Test_CheckdirCategory_to "ERROR: ~/archivers/Makefile:4: The file should end here.") } -func (s *Suite) Test_CheckdirCategory_invalid_comment(c *check.C) { +func (s *Suite) Test_CheckdirCategory__invalid_comment(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -53,3 +53,33 @@ func (s *Suite) Test_CheckdirCategory_in t.CheckOutputLines( "WARN: ~/archivers/Makefile:2: COMMENT contains invalid characters (U+005C U+0024 U+0024 U+0024 U+0024 U+0022).") } + +func (s *Suite) Test_CheckdirCategory__wip(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.SetupVartypes() + t.SetupFileLines("mk/misc/category.mk") + t.SetupFileLines("wip/package/Makefile") + t.SetupFileLines("wip/fs-only/Makefile") + t.SetupFileLines("wip/Makefile", + MkRcsID, + "COMMENT=\tCategory comment", + "", + "SUBDIR+=\tmk-only", + "#SUBDIR+=\tpackage", + "SUBDIR+=\tpackage", + "", + "wip-specific-target: .PHONY", + "\t${RUN}wip-specific-command", + "", + ".include \"../mk/misc/category.mk\"") + + G.CheckDirent(t.File("wip")) + + t.CheckOutputLines( + "WARN: ~/wip/Makefile:5: \"package\" commented out without giving a reason.", + "ERROR: ~/wip/Makefile:6: \"package\" must only appear once.", + "ERROR: ~/wip/Makefile:4: \"fs-only\" exists in the file system, but not in the Makefile.", + "ERROR: ~/wip/Makefile:4: \"mk-only\" exists in the Makefile, but not in the file system.") +} Index: pkgsrc/pkgtools/pkglint/files/expecter.go diff -u pkgsrc/pkgtools/pkglint/files/expecter.go:1.11 pkgsrc/pkgtools/pkglint/files/expecter.go:1.12 --- pkgsrc/pkgtools/pkglint/files/expecter.go:1.11 Thu Aug 9 20:08:12 2018 +++ pkgsrc/pkgtools/pkglint/files/expecter.go Wed Sep 5 17:56:22 2018 @@ -82,16 +82,6 @@ func (exp *Expecter) AdvanceIfEquals(tex return !exp.EOF() && exp.lines[exp.index].Text == text && exp.Advance() } -func (exp *Expecter) AdvanceWhile(pred func(line Line) bool) { - if trace.Tracing { - defer trace.Call(exp.CurrentLine().Text)() - } - - for !exp.EOF() && !pred(exp.CurrentLine()) { - exp.Advance() - } -} - func (exp *Expecter) ExpectEmptyLine(warnSpace bool) bool { if exp.AdvanceIfEquals("") { return true @@ -135,10 +125,6 @@ func (exp *MkExpecter) CurrentMkLine() M return exp.mklines.mklines[exp.index] } -func (exp *MkExpecter) PreviousMkLine() MkLine { - return exp.mklines.mklines[exp.index-1] -} - func (exp *MkExpecter) AdvanceWhile(pred func(mkline MkLine) bool) { if trace.Tracing { defer trace.Call(exp.CurrentMkLine().Text)() Index: pkgsrc/pkgtools/pkglint/files/check_test.go diff -u pkgsrc/pkgtools/pkglint/files/check_test.go:1.24 pkgsrc/pkgtools/pkglint/files/check_test.go:1.25 --- pkgsrc/pkgtools/pkglint/files/check_test.go:1.24 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/check_test.go Wed Sep 5 17:56:22 2018 @@ -5,6 +5,7 @@ import ( "fmt" "io" "io/ioutil" + "netbsd.org/pkglint/regex" "os" "path" "path/filepath" @@ -27,13 +28,23 @@ type Suite struct { Tester *Tester } -// Init initializes the suite with the check.C instance for the actual -// test run. -// The returned tester can be used to easily setup the test environment -// and check the results using a high-level API. +// Init creates and returns a test helper that allows to: // -// See https://github.com/go-check/check/issues/22 +// * create files for the test +// +// * load these files into Line and MkLine objects (for tests spanning multiple files) +// +// * create new in-memory Line and MkLine objects (for simple tests) +// +// * check the files that have been changed by the --autofix feature +// +// * check the pkglint diagnostics func (s *Suite) Init(c *check.C) *Tester { + + // Note: the check.C object from SetUpTest cannot be used here, + // and the parameter given here cannot be used in TearDownTest; + // see https://github.com/go-check/check/issues/22. + t := s.Tester // Has been initialized by SetUpTest if t.checkC != nil { panic("Suite.Init must only be called once.") @@ -149,19 +160,17 @@ func (t *Tester) SetupOption(name, descr G.Pkgsrc.PkgOptions[name] = description } -func (t *Tester) SetupTool(tool *Tool) { - reg := G.Pkgsrc.Tools +func (t *Tester) SetupTool(name, varname string) *Tool { + tools := G.Pkgsrc.Tools + return tools.Define(name, varname, dummyMkLine) +} - if len(reg.byName) == 0 && len(reg.byVarname) == 0 { - reg = NewToolRegistry() - G.Pkgsrc.Tools = reg - } - if tool.Name != "" { - reg.byName[tool.Name] = tool - } - if tool.Varname != "" { - reg.byVarname[tool.Varname] = tool - } +// SetupToolUsable registers a tool and immediately makes it usable, +// as if the tool were predefined globally in pkgsrc. +func (t *Tester) SetupToolUsable(name, varname string) *Tool { + tool := t.SetupTool(name, varname) + tool.SetValidity(AtRunTime, G.Pkgsrc.Tools.TraceName) + return tool } // SetupFileLines creates a temporary file and writes the given lines to it. @@ -197,6 +206,13 @@ func (t *Tester) SetupPkgsrc() { t.CreateFileLines("doc/TODO", RcsID) + // Some example licenses so that the tests for whole packages + // don't need to define them on their own. + t.CreateFileLines("licenses/2-clause-bsd", + "Redistribution and use in source and binary forms ...") + t.CreateFileLines("licenses/gnu-gpl-v2", + "The licenses for most software ...") + // The MASTER_SITES in the package Makefile are searched here. // See Pkgsrc.loadMasterSites. t.CreateFileLines("mk/fetch/sites.mk", @@ -279,25 +295,49 @@ func (t *Tester) Chdir(relativeFilename t.relcwd = relativeFilename } -// ExpectFatalError promises that in the remainder of the current function -// call, a panic with a pkglintFatal will occur (typically from Line.Fatalf). +// ExpectFatal runs the given action and expects that this action calls +// Line.Fatalf or uses some other way to panic with a pkglintFatal. // // Usage: -// func() { -// defer t.ExpectFatalError() +// t.ExpectFatal( +// func() { /* do something that panics */ }, +// "FATAL: ~/Makefile:1: Must not be empty") +func (t *Tester) ExpectFatal(action func(), expectedLines ...string) { + defer func() { + r := recover() + if r == nil { + panic("Expected a pkglint fatal error, but didn't get one.") + } else if _, ok := r.(pkglintFatal); ok { + t.CheckOutputLines(expectedLines...) + } else { + panic(r) + } + }() + + action() +} + +// ExpectFatalMatches runs the given action and expects that this action +// calls Line.Fatalf or uses some other way to panic with a pkglintFatal. +// It then matches the output against a regular expression. // -// // The code that causes the fatal error. -// Load(t.File("nonexistent"), MustSucceed) -// }() -// t.CheckOutputLines( -// "FATAL: ~/nonexistent: Does not exist.") -func (t *Tester) ExpectFatalError() { - r := recover() - if r == nil { - panic("Expected a pkglint fatal error, but didn't get one.") - } else if _, ok := r.(pkglintFatal); !ok { - panic(r) - } +// Usage: +// t.ExpectFatalMatches( +// func() { /* do something that panics */ }, +// `FATAL: ~/Makefile:1: .*\n`) +func (t *Tester) ExpectFatalMatches(action func(), expected regex.Pattern) { + defer func() { + r := recover() + if r == nil { + panic("Expected a pkglint fatal error, but didn't get one.") + } else if _, ok := r.(pkglintFatal); ok { + t.c().Check(t.Output(), check.Matches, string(expected)) + } else { + panic(r) + } + }() + + action() } // Arguments are either (lineno, orignl) or (lineno, orignl, textnl). @@ -397,6 +437,9 @@ func (t *Tester) CheckOutputLines(expect // in an in-memory buffer) additionally to stdout. // This is useful when stepping through the code, especially // in combination with SetupCommandLine("--debug"). +// +// In JetBrains GoLand, the tracing output is suppressed after the first +// failed check, see https://youtrack.jetbrains.com/issue/GO-6154. func (t *Tester) EnableTracing() { G.logOut = NewSeparatorWriter(io.MultiWriter(os.Stdout, &t.stdout)) trace.Out = os.Stdout Index: pkgsrc/pkgtools/pkglint/files/line.go diff -u pkgsrc/pkgtools/pkglint/files/line.go:1.24 pkgsrc/pkgtools/pkglint/files/line.go:1.25 --- pkgsrc/pkgtools/pkglint/files/line.go:1.24 Thu Jul 12 16:23:36 2018 +++ pkgsrc/pkgtools/pkglint/files/line.go Wed Sep 5 17:56:22 2018 @@ -33,6 +33,7 @@ func (rline *RawLine) String() string { type LineImpl struct { Filename string + Basename string firstLine int32 // Zero means not applicable, -1 means EOF lastLine int32 // Usually the same as firstLine, may differ in Makefiles Text string @@ -46,7 +47,7 @@ func NewLine(fname string, lineno int, t // NewLineMulti is for logical Makefile lines that end with backslash. func NewLineMulti(fname string, firstLine, lastLine int, text string, rawLines []*RawLine) Line { - return &LineImpl{fname, int32(firstLine), int32(lastLine), text, rawLines, nil} + return &LineImpl{fname, path.Base(fname), int32(firstLine), int32(lastLine), text, rawLines, nil} } // NewLineEOF creates a dummy line for logging, with the "line number" EOF. Index: pkgsrc/pkgtools/pkglint/files/plist_test.go diff -u pkgsrc/pkgtools/pkglint/files/plist_test.go:1.24 pkgsrc/pkgtools/pkglint/files/plist_test.go:1.25 --- pkgsrc/pkgtools/pkglint/files/plist_test.go:1.24 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/plist_test.go Wed Sep 5 17:56:22 2018 @@ -38,6 +38,7 @@ func (s *Suite) Test_ChecklinesPlist(c * "ERROR: PLIST:6: \"info/dir\" must not be listed. Use install-info to add/remove an entry.", "WARN: PLIST:8: Redundant library found. The libtool library is in line 9.", "WARN: PLIST:9: \"lib/libc.la\" should be sorted before \"lib/libc.so.6\".", + "WARN: PLIST:9: Packages that install libtool libraries should define USE_LIBTOOL.", "WARN: PLIST:10: Preformatted manual page without unformatted one.", "WARN: PLIST:10: Preformatted manual pages should end in \".0\".", "WARN: PLIST:11: IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.", @@ -336,3 +337,240 @@ func (s *Suite) Test_PlistChecker__autof "sbin/program", "bin/program") } + +func (s *Suite) Test_PlistChecker__exec_MKDIR(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "bin/program", + "@exec ${MKDIR} %D/share/mk/subdir") + + ChecklinesPlist(lines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_PlistChecker__empty_line(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "", + "bin/program") + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "WARN: ~/PLIST:2: PLISTs should not contain empty lines.") + + t.SetupCommandLine("-Wall", "--autofix") + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "AUTOFIX: ~/PLIST:2: Deleting this line.") + t.CheckFileLines("PLIST", + PlistRcsID, + "bin/program") +} + +func (s *Suite) Test_PlistChecker__unknown_line_type(c *check.C) { + t := s.Init(c) + + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "---unknown", + "+++unknown") + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "WARN: ~/PLIST:2: Unknown line type: ---unknown", + "WARN: ~/PLIST:3: Unknown line type: +++unknown") +} + +func (s *Suite) Test_PlistChecker__doc(c *check.C) { + t := s.Init(c) + + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "doc/html/index.html") + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "ERROR: ~/PLIST:2: Documentation must be installed under share/doc, not doc.") +} + +func (s *Suite) Test_PlistChecker__PKGLOCALEDIR(c *check.C) { + t := s.Init(c) + + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "${PKGLOCALEDIR}/file") + G.Pkg = NewPackage(t.File("category/package")) + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "WARN: ~/PLIST:2: PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR was not found.") +} + +func (s *Suite) Test_PlistChecker__unwanted_entries(c *check.C) { + t := s.Init(c) + + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "share/pkgbase/CVS/Entries", + "share/pkgbase/Makefile.orig", + "share/perllocal.pod") + G.Pkg = NewPackage(t.File("category/package")) + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "WARN: ~/PLIST:2: CVS files should not be in the PLIST.", + "WARN: ~/PLIST:3: .orig files should not be in the PLIST.", + "WARN: ~/PLIST:4: perllocal.pod files should not be in the PLIST.") +} + +func (s *Suite) Test_PlistChecker_checkpathInfo(c *check.C) { + t := s.Init(c) + + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "info/gmake.1.info") + G.Pkg = NewPackage(t.File("category/package")) + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "WARN: ~/PLIST:2: Packages that install info files should set INFO_FILES in the Makefile.") +} + +func (s *Suite) Test_PlistChecker_checkpathLib(c *check.C) { + t := s.Init(c) + + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "lib/package/liberty-1.0.so", + "lib/charset.alias", + "lib/locale/de_DE/liberty.mo", + "lib/liberty-1.0.la") + G.Pkg = NewPackage(t.File("category/package")) + G.Pkg.EffectivePkgbase = "package" + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "ERROR: ~/PLIST:3: Only the libiconv package may install lib/charset.alias.", + "ERROR: ~/PLIST:4: \"lib/locale\" must not be listed. Use ${PKGLOCALEDIR}/locale and set USE_PKGLOCALEDIR instead.", + "WARN: ~/PLIST:5: Packages that install libtool libraries should define USE_LIBTOOL.") +} + +func (s *Suite) Test_PlistChecker_checkpathMan(c *check.C) { + t := s.Init(c) + + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "man/manx/program.x", + "man/man1/program.8") + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "WARN: ~/PLIST:2: Unknown section \"x\" for manual page.", + "WARN: ~/PLIST:3: Mismatch between the section (1) and extension (8) of the manual page.") +} + +func (s *Suite) Test_PlistChecker_checkpathShare(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "share/doc/html/package/index.html", + "share/doc/package/index.html", + "share/icons/hicolor/icon-theme.cache", + "share/info/program.1.info", + "share/man/man1/program.1") + G.Pkg = NewPackage(t.File("category/package")) + G.Pkg.EffectivePkgbase = "package" + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "WARN: ~/PLIST:2: Use of \"share/doc/html\" is deprecated. Use \"share/doc/${PKGBASE}\" instead.", + "ERROR: ~/PLIST:4: Packages that install hicolor icons must include \"../../graphics/hicolor-icon-theme/buildlink3.mk\" in the Makefile.", + "ERROR: ~/PLIST:4: The file icon-theme.cache must not appear in any PLIST file.", + "WARN: ~/PLIST:4: Packages that install icon theme files should set ICON_THEMES.", + "WARN: ~/PLIST:5: Info pages should be installed into info/, not share/info/.", + "WARN: ~/PLIST:6: Man pages should be installed into man/, not share/man/.") +} + +func (s *Suite) Test_PlistLine_CheckTrailingWhitespace(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "bin/program \t") + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "ERROR: ~/PLIST:2: pkgsrc does not support filenames ending in white-space.") +} + +func (s *Suite) Test_PlistLine_CheckDirective(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "@unexec rmdir %D/bin", + "@exec ldconfig", + "@comment This is a comment", + "@dirrm %D/bin", + "@imake-man 1 2 3 4", + "@imake-man 1 2 ${IMAKE_MANNEWSUFFIX}", + "@unknown") + + ChecklinesPlist(lines) + + t.CheckOutputLines( + "WARN: ~/PLIST:2: Please remove this line. It is no longer necessary.", + "ERROR: ~/PLIST:3: ldconfig must be used with \"||/usr/bin/true\".", + "WARN: ~/PLIST:5: @dirrm is obsolete. Please remove this line.", + "WARN: ~/PLIST:6: Invalid number of arguments for imake-man.", + "WARN: ~/PLIST:7: IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.", + "WARN: ~/PLIST:8: Unknown PLIST directive \"@unknown\".") +} + +func (s *Suite) Test_plistLineSorter__unsortable(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall", "--show-autofix") + lines := t.SetupFileLines("PLIST", + PlistRcsID, + "bin/program${OPSYS}", + "@exec true", + "bin/program1") + + t.EnableTracingToLog() + ChecklinesPlist(lines) + + t.CheckOutputLines( + "TRACE: + ChecklinesPlist(\"~/PLIST\")", + "TRACE: 1 + CheckLineRcsid(\"@comment \", \"@comment \")", + "TRACE: 1 - CheckLineRcsid(\"@comment \", \"@comment \")", + "TRACE: 1 ~/PLIST:2: bin/program${OPSYS}: This line prevents pkglint from sorting the PLIST automatically.", + "TRACE: 1 + SaveAutofixChanges()", + "TRACE: 1 - SaveAutofixChanges()", + "TRACE: - ChecklinesPlist(\"~/PLIST\")") +} Index: pkgsrc/pkgtools/pkglint/files/shell.go diff -u pkgsrc/pkgtools/pkglint/files/shell.go:1.24 pkgsrc/pkgtools/pkglint/files/shell.go:1.25 --- pkgsrc/pkgtools/pkglint/files/shell.go:1.24 Sun Aug 12 16:31:56 2018 +++ pkgsrc/pkgtools/pkglint/files/shell.go Wed Sep 5 17:56:22 2018 @@ -13,7 +13,7 @@ const ( reShVarname = `(?:[!#*\-\d?@]|\$\$|[A-Za-z_]\w*)` reShVarexpansion = `(?:(?:#|##|%|%%|:-|:=|:\?|:\+|\+)[^$\\{}]*)` reShVaruse = `\$\$` + `(?:` + reShVarname + `|` + `\{` + reShVarname + `(?:` + reShVarexpansion + `)?` + `\})` - reShDollar = `\\\$\$|` + reShVaruse + `|\$\$[,\-/|]` + reShDollar = `\\\$\$|` + reShVaruse + `|\$\$[,\-/]` ) type ShellLine struct { @@ -27,7 +27,7 @@ func NewShellLine(mkline MkLine) *ShellL var shellcommandsContextType = &Vartype{lkNone, BtShellCommands, []ACLEntry{{"*", aclpAllRuntime}}, false} var shellwordVuc = &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucQuotPlain, false} -func (shline *ShellLine) CheckWord(token string, checkQuoting bool) { +func (shline *ShellLine) CheckWord(token string, checkQuoting bool, time ToolTime) { if trace.Tracing { defer trace.Call(token, checkQuoting)() } @@ -38,6 +38,8 @@ func (shline *ShellLine) CheckWord(token var line = shline.mkline.Line + // Delegate check for shell words consisting of a single variable use + // to the MkLineChecker. Examples for these are ${VAR:Mpattern} or $@. p := NewMkParser(line, token, false) if varuse := p.VarUse(); varuse != nil && p.EOF() { MkLineChecker{shline.mkline}.CheckVaruse(varuse, shellwordVuc) @@ -69,7 +71,7 @@ outer: var backtCommand string backtCommand, quoting = shline.unescapeBackticks(token, repl, quoting) setE := true - shline.CheckShellCommand(backtCommand, &setE) + shline.CheckShellCommand(backtCommand, &setE, time) // Make(1) variables have the same syntax, no matter in which state we are currently. case shline.checkVaruseToken(parser, quoting): @@ -106,11 +108,6 @@ outer: "\tcp \"$fname\" /tmp", "\t# copies one file, as intended") } - case repl.AdvanceStr("$@"): - line.Warnf("Please use %q instead of %q.", "${.TARGET}", "$@") - Explain( - "It is more readable and prevents confusion with the shell variable of", - "the same name.") case repl.AdvanceStr("$$@"): line.Warnf("The $@ shell variable should only be used in double quotes.") @@ -123,6 +120,7 @@ outer: Explain( "The Solaris /bin/sh does not know this way to execute a command in a", "subshell. Please use backticks (`...`) as a replacement.") + return // To avoid internal parse errors case repl.AdvanceStr("$$"): // Not part of a variable. break @@ -167,7 +165,7 @@ outer: } if strings.TrimSpace(parser.Rest()) != "" { - line.Warnf("Pkglint parse error in ShellLine.CheckWord at %q (quoting=%s, rest=%q)", token, quoting, parser.Rest()) + line.Warnf("Pkglint parse error in ShellLine.CheckWord at %q (quoting=%s), rest: %s", token, quoting, parser.Rest()) } } @@ -220,7 +218,7 @@ func (shline *ShellLine) checkVaruseToke // See http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_06_03 func (shline *ShellLine) unescapeBackticks(shellword string, repl *textproc.PrefixReplacer, quoting ShQuoting) (unescaped string, newQuoting ShQuoting) { if trace.Tracing { - defer trace.Call(shellword, quoting, "=>", trace.Ref(&unescaped))() + defer trace.Call(shellword, quoting, trace.Result(&unescaped))() } line := shline.mkline.Line @@ -234,8 +232,8 @@ func (shline *ShellLine) unescapeBacktic } return unescaped, quoting - case repl.AdvanceRegexp("^\\\\([\"\\\\`$])"): - unescaped += repl.Group(1) + case repl.AdvanceStr("\\\""), repl.AdvanceStr("\\\\"), repl.AdvanceStr("\\`"), repl.AdvanceStr("\\$"): + unescaped += repl.Str()[1:] case repl.AdvanceStr("\\"): line.Warnf("Backslashes should be doubled inside backticks.") @@ -247,13 +245,15 @@ func (shline *ShellLine) unescapeBacktic "According to the SUSv3, they produce undefined results.", "", "See the paragraph starting \"Within the backquoted ...\" in", - "http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html") + "http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html.", + "", + "To avoid this uncertainty, escape the double quotes using \\\".") case repl.AdvanceRegexp("^([^\\\\`]+)"): unescaped += repl.Group(1) default: - line.Errorf("Internal pkglint error in ShellLine.unescapeBackticks at %q (rest=%q)", shellword, repl.Rest()) + line.Errorf("Internal pkglint error in ShellLine.unescapeBackticks at %q (rest=%q).", shellword, repl.AdvanceRest()) } } line.Errorf("Unfinished backquotes: rest=%q", repl.Rest()) @@ -316,10 +316,10 @@ func (shline *ShellLine) CheckShellComma repl.AdvanceStr("${_PKG_SILENT}${_PKG_DEBUG}") } - shline.CheckShellCommand(repl.Rest(), &setE) + shline.CheckShellCommand(repl.Rest(), &setE, RunTime) } -func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool) { +func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool, time ToolTime) { if trace.Tracing { defer trace.Call()() } @@ -340,7 +340,7 @@ func (shline *ShellLine) CheckShellComma callback := NewMkShWalkCallback() callback.SimpleCommand = func(command *MkShSimpleCommand) { - scc := NewSimpleCommandChecker(shline, command) + scc := NewSimpleCommandChecker(shline, command, time) scc.Check() if scc.strcmd.Name == "set" && scc.strcmd.AnyArgMatches(`^-.*e`) { *pSetE = true @@ -353,15 +353,15 @@ func (shline *ShellLine) CheckShellComma spc.checkPipeExitcode(line, pipeline) } callback.Word = func(word *ShToken) { - spc.checkWord(word, false) + spc.checkWord(word, false, time) } NewMkShWalker().Walk(program, callback) } -func (shline *ShellLine) CheckShellCommands(shellcmds string) { +func (shline *ShellLine) CheckShellCommands(shellcmds string, time ToolTime) { setE := true - shline.CheckShellCommand(shellcmds, &setE) + shline.CheckShellCommand(shellcmds, &setE, time) if !hasSuffix(shellcmds, ";") { shline.mkline.Warnf("This shell command list should end with a semicolon.") } @@ -421,11 +421,12 @@ type SimpleCommandChecker struct { shline *ShellLine cmd *MkShSimpleCommand strcmd *StrCommand + time ToolTime } -func NewSimpleCommandChecker(shline *ShellLine, cmd *MkShSimpleCommand) *SimpleCommandChecker { +func NewSimpleCommandChecker(shline *ShellLine, cmd *MkShSimpleCommand, time ToolTime) *SimpleCommandChecker { strcmd := NewStrCommand(cmd) - return &SimpleCommandChecker{shline, cmd, strcmd} + return &SimpleCommandChecker{shline, cmd, strcmd, time} } @@ -468,30 +469,27 @@ func (scc *SimpleCommandChecker) checkCo } } +// handleTool tests whether the shell command is one of the recognized pkgsrc tools +// and whether the package has added it to USE_TOOLS. func (scc *SimpleCommandChecker) handleTool() bool { if trace.Tracing { defer trace.Call()() } - shellword := scc.strcmd.Name - tool, localTool := G.Pkgsrc.Tools.ByName(shellword), false - if tool == nil && G.Mk != nil { - tool, localTool = G.Mk.toolRegistry.byName[shellword], true - } - if tool == nil { - return false - } + command := scc.strcmd.Name + + tool, usable := G.Tool(command, scc.time) - if !localTool && !G.Mk.tools[shellword] && !G.Mk.tools["g"+shellword] { - scc.shline.mkline.Warnf("The %q tool is used but not added to USE_TOOLS.", shellword) + if tool != nil && !usable { + scc.shline.mkline.Warnf("The %q tool is used but not added to USE_TOOLS.", command) } - if tool.MustUseVarForm { - scc.shline.mkline.Warnf("Please use \"${%s}\" instead of %q.", tool.Varname, shellword) + if tool != nil && !containsVarRef(command) && tool.MustUseVarForm { + scc.shline.mkline.Warnf("Please use \"${%s}\" instead of %q.", tool.Varname, command) } - scc.shline.checkCommandUse(shellword) - return true + scc.shline.checkCommandUse(command) + return tool != nil } func (scc *SimpleCommandChecker) handleForbiddenCommand() bool { @@ -505,8 +503,8 @@ func (scc *SimpleCommandChecker) handleF scc.shline.mkline.Errorf("%q must not be used in Makefiles.", shellword) Explain( "This command must appear in INSTALL scripts, not in the package", - "Makefile, so that the package also works if it is installed as a binary", - "package via pkg_add.") + "Makefile, so that the package also works if it is installed as a", + "binary package via pkg_add.") return true } return false @@ -522,15 +520,15 @@ func (scc *SimpleCommandChecker) handleC if varuse := parser.VarUse(); varuse != nil && parser.EOF() { varname := varuse.varname - if tool := G.Pkgsrc.Tools.ByVarname(varname); tool != nil { - if !G.Mk.tools[tool.Name] { + if tool := G.ToolByVarname(varname, RunTime /* LoadTime would also work */); tool != nil { + if tool.Validity == Nowhere { scc.shline.mkline.Warnf("The %q tool is used but not added to USE_TOOLS.", tool.Name) } scc.shline.checkCommandUse(shellword) return true } - if vartype := scc.shline.mkline.VariableType(varname); vartype != nil && vartype.basicType.name == "ShellCommand" { + if vartype := G.Pkgsrc.VariableType(varname); vartype != nil && vartype.basicType.name == "ShellCommand" { scc.shline.checkCommandUse(shellword) return true } @@ -692,12 +690,12 @@ func (scc *SimpleCommandChecker) checkPa defer trace.Call()() } - if scc.strcmd.Name == "${PAX}" && scc.strcmd.HasOption("-pe") { + if (scc.strcmd.Name == "${PAX}" || scc.strcmd.Name == "pax") && scc.strcmd.HasOption("-pe") { scc.shline.mkline.Warnf("Please use the -pp option to pax(1) instead of -pe.") Explain( - "The -pe option tells pax to preserve the ownership of the files, which", - "means that the installed files will belong to the user that has built", - "the package.") + "The -pe option tells pax to preserve the ownership of the files,", + "which means that the installed files will belong to the user that", + "has built the package.") } } @@ -766,22 +764,22 @@ func (spc *ShellProgramChecker) checkCon NewMkShWalker().Walk(list, callback) } -func (spc *ShellProgramChecker) checkWords(words []*ShToken, checkQuoting bool) { +func (spc *ShellProgramChecker) checkWords(words []*ShToken, checkQuoting bool, time ToolTime) { if trace.Tracing { defer trace.Call()() } for _, word := range words { - spc.checkWord(word, checkQuoting) + spc.checkWord(word, checkQuoting, time) } } -func (spc *ShellProgramChecker) checkWord(word *ShToken, checkQuoting bool) { +func (spc *ShellProgramChecker) checkWord(word *ShToken, checkQuoting bool, time ToolTime) { if trace.Tracing { defer trace.Call(word.MkText)() } - spc.shline.CheckWord(word.MkText, checkQuoting) + spc.shline.CheckWord(word.MkText, checkQuoting, time) } func (spc *ShellProgramChecker) checkPipeExitcode(line Line, pipeline *MkShPipeline) { @@ -810,18 +808,19 @@ func (spc *ShellProgramChecker) checkPip if simple == nil { return true, "" } + commandName := simple.Name.MkText if len(simple.Redirections) != 0 { - return true, simple.Name.MkText + return true, commandName } - tool := G.Pkgsrc.Tools.FindByCommand(simple.Name) + tool, _ := G.Tool(commandName, RunTime) switch { case tool == nil: - return true, simple.Name.MkText + return true, commandName case oneOf(tool.Name, "echo", "printf"): case oneOf(tool.Name, "sed", "gsed", "grep", "ggrep") && len(simple.Args) == 1: break default: - return true, simple.Name.MkText + return true, commandName } } return false, "" @@ -921,17 +920,28 @@ func splitIntoShellTokens(line Line, tex } word := "" + p := NewShTokenizer(line, text, false) emit := func() { if word != "" { tokens = append(tokens, word) word = "" } + rest = p.mkp.Rest() } - p := NewShTokenizer(line, text, false) - atoms := p.ShAtoms() + q := shqPlain - for _, atom := range atoms { + var prevAtom *ShAtom + for { + atom := p.ShAtom(q) + if atom == nil { + if prevAtom == nil || prevAtom.Quoting == shqPlain { + emit() + } + break + } + q = atom.Quoting + prevAtom = atom if atom.Type == shtSpace && q == shqPlain { emit() } else if atom.Type == shtWord || atom.Type == shtVaruse || atom.Quoting != shqPlain { @@ -941,8 +951,8 @@ func splitIntoShellTokens(line Line, tex tokens = append(tokens, atom.MkText) } } - emit() - return tokens, word + p.mkp.Rest() + + return } // Example: "word1 word2;;;" => "word1", "word2;;;" Index: pkgsrc/pkgtools/pkglint/files/licenses.go diff -u pkgsrc/pkgtools/pkglint/files/licenses.go:1.13 pkgsrc/pkgtools/pkglint/files/licenses.go:1.14 --- pkgsrc/pkgtools/pkglint/files/licenses.go:1.13 Sun Aug 12 16:31:56 2018 +++ pkgsrc/pkgtools/pkglint/files/licenses.go Wed Sep 5 17:56:22 2018 @@ -45,10 +45,10 @@ func (lc *LicenseChecker) Check(value st } func (lc *LicenseChecker) checkLicenseName(license string) { - var licenseFile string + licenseFile := "" if G.Pkg != nil { - if licenseFileValue, ok := G.Pkg.varValue("LICENSE_FILE"); ok { - licenseFile = G.Pkg.File(lc.MkLine.ResolveVarsInRelativePath(licenseFileValue, false)) + if mkline := G.Pkg.vars.FirstDefinition("LICENSE_FILE"); mkline != nil { + licenseFile = G.Pkg.File(mkline.ResolveVarsInRelativePath(mkline.Value(), false)) } } if licenseFile == "" { Index: pkgsrc/pkgtools/pkglint/files/logging.go diff -u pkgsrc/pkgtools/pkglint/files/logging.go:1.13 pkgsrc/pkgtools/pkglint/files/logging.go:1.14 --- pkgsrc/pkgtools/pkglint/files/logging.go:1.13 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/logging.go Wed Sep 5 17:56:22 2018 @@ -62,6 +62,7 @@ func logs(level *LogLevel, fname, lineno } if !G.opts.LogVerbose && loggedAlready(fname, lineno, msg) { + G.explainNext = false return false } @@ -112,11 +113,11 @@ func Explain(explanation ...string) { for _, s := range explanation { if l := tabWidth(s); l > 68 && contains(s, " ") { lastSpace := strings.LastIndexByte(s[:68], ' ') - print(fmt.Sprintf("Long explanation line: %s\nBreak after: %s\n", s, s[:lastSpace])) + G.logErr.Write(fmt.Sprintf("Long explanation line: %s\nBreak after: %s\n", s, s[:lastSpace])) } if m, before := match1(s, `(.+)\. [^ ]`); m { if !matches(before, `\d$|e\.g`) { - print(fmt.Sprintf("Short space after period: %s\n", s)) + G.logErr.Write(fmt.Sprintf("Short space after period: %s\n", s)) } } } @@ -150,8 +151,8 @@ func Explain(explanation ...string) { type pkglintFatal struct{} // SeparatorWriter writes output, occasionally separated by an -// empty line. This is used for layouting the diagnostics in -// --source mode combined with --show-autofix, where each +// empty line. This is used for separating the diagnostics when +// --source is combined with --show-autofix, where each // log message consists of multiple lines. type SeparatorWriter struct { out io.Writer Index: pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go diff -u pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go:1.13 pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go:1.14 --- pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go:1.13 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/mklinechecker_test.go Wed Sep 5 17:56:22 2018 @@ -7,23 +7,22 @@ func (s *Suite) Test_MkLineChecker_Check t.SetupCommandLine("-Wtypes") t.SetupVartypes() - mkline := t.NewMkLine("fname", 1, "COMMENT=\tA nice package") vartype1 := G.Pkgsrc.vartypes["COMMENT"] c.Assert(vartype1, check.NotNil) c.Check(vartype1.guessed, equals, false) - vartype := mkline.VariableType("COMMENT") + vartype := G.Pkgsrc.VariableType("COMMENT") c.Assert(vartype, check.NotNil) c.Check(vartype.basicType.name, equals, "Comment") c.Check(vartype.guessed, equals, false) c.Check(vartype.kindOfList, equals, lkNone) - MkLineChecker{mkline}.CheckVartype("COMMENT", opAssign, "A nice package", "") + MkLineChecker{dummyMkLine}.CheckVartype("COMMENT", opAssign, "A nice package", "") t.CheckOutputLines( - "WARN: fname:1: COMMENT should not begin with \"A\".") + "WARN: COMMENT should not begin with \"A\".") } func (s *Suite) Test_MkLineChecker_CheckVartype(c *check.C) { @@ -57,52 +56,41 @@ func (s *Suite) Test_MkLineChecker_Check t.SetupCommandLine("-Wtypes") t.SetupVartypes() - MkLineChecker{t.NewMkLine("fname", 1, ".if !empty(PKGSRC_COMPILER:Mmycc)")}.checkDirectiveCond() + testCond := func(cond string, output ...string) { + MkLineChecker{t.NewMkLine("fname", 1, cond)}.checkDirectiveCond() + t.CheckOutputLines(output...) + } - t.CheckOutputLines( - "WARN: fname:1: The pattern \"mycc\" cannot match any of " + - "{ ccache ccc clang distcc f2c gcc hp icc ido " + + testCond(".if !empty(PKGSRC_COMPILER:Mmycc)", + "WARN: fname:1: The pattern \"mycc\" cannot match any of "+ + "{ ccache ccc clang distcc f2c gcc hp icc ido "+ "mipspro mipspro-ucode pcc sunpro xlc } for PKGSRC_COMPILER.") - MkLineChecker{t.NewMkLine("fname", 1, ".elif ${A} != ${B}")}.checkDirectiveCond() - - t.CheckOutputEmpty() - - MkLineChecker{t.NewMkLine("fname", 1, ".if ${HOMEPAGE} == \"mailto:someone@example.org\"")}.checkDirectiveCond() + testCond(".elif ${A} != ${B}") - t.CheckOutputLines( + testCond(".if ${HOMEPAGE} == \"mailto:someone@example.org\"", "WARN: fname:1: \"mailto:someone@example.org\" is not a valid URL.") - MkLineChecker{t.NewMkLine("fname", 1, ".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])")}.checkDirectiveCond() - - t.CheckOutputLines( + testCond(".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])", "WARN: fname:1: PKGSRC_RUN_TEST should be matched against \"[yY][eE][sS]\" or \"[nN][oO]\", not \"[Y][eE][sS]\".") - MkLineChecker{t.NewMkLine("fname", 1, ".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])")}.checkDirectiveCond() - - t.CheckOutputEmpty() - - MkLineChecker{t.NewMkLine("fname", 1, ".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})")}.checkDirectiveCond() + testCond(".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])") - t.CheckOutputLines( + testCond(".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})", "WARN: fname:1: The empty() function takes a variable name as parameter, not a variable expression.") - MkLineChecker{t.NewMkLine("fname", 1, ".if ${EMUL_PLATFORM} == \"linux-x386\"")}.checkDirectiveCond() - - t.CheckOutputLines( - "WARN: fname:1: " + - "\"x386\" is not valid for the hardware architecture part of EMUL_PLATFORM. " + - "Use one of " + - "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex " + - "dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb " + - "earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 " + - "i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 " + - "mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 " + + testCond(".if ${EMUL_PLATFORM} == \"linux-x386\"", + "WARN: fname:1: "+ + "\"x386\" is not valid for the hardware architecture part of EMUL_PLATFORM. "+ + "Use one of "+ + "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex "+ + "dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb "+ + "earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 "+ + "i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 "+ + "mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} instead.") - MkLineChecker{t.NewMkLine("fname", 1, ".if ${EMUL_PLATFORM:Mlinux-x386}")}.checkDirectiveCond() - - t.CheckOutputLines( + testCond(".if ${EMUL_PLATFORM:Mlinux-x386}", "WARN: fname:1: "+ "The pattern \"x386\" cannot match any of { aarch64 aarch64eb alpha amd64 arc arm arm26 "+ "arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb "+ @@ -113,15 +101,13 @@ func (s *Suite) Test_MkLineChecker_Check "for the hardware architecture part of EMUL_PLATFORM.", "NOTE: fname:1: EMUL_PLATFORM should be compared using == instead of the :M or :N modifier without wildcards.") - MkLineChecker{t.NewMkLine("fname", 98, ".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}")}.checkDirectiveCond() - - t.CheckOutputLines( - "WARN: fname:98: "+ + testCond(".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", + "WARN: fname:1: "+ "The pattern \"UnknownOS\" cannot match any of "+ "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ "} for the operating system part of MACHINE_PLATFORM.", - "WARN: fname:98: "+ + "WARN: fname:1: "+ "The pattern \"x86\" cannot match any of "+ "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm "+ "earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb "+ @@ -129,7 +115,9 @@ func (s *Suite) Test_MkLineChecker_Check "m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax "+ "powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} for MACHINE_ARCH.", - "NOTE: fname:98: MACHINE_ARCH should be compared using == instead of the :M or :N modifier without wildcards.") + "NOTE: fname:1: MACHINE_ARCH should be compared using == instead of the :M or :N modifier without wildcards.") + + testCond(".if ${MASTER_SITES:Mftp://*} == \"ftp://netbsd.org/\"") } func (s *Suite) Test_MkLineChecker_checkVarassign(c *check.C) { @@ -147,14 +135,14 @@ func (s *Suite) Test_MkLineChecker_check "WARN: Makefile:2: ac_cv_libpari_libs is defined but not used.") } -func (s *Suite) Test_MkLineChecker_checkVarassignDefPermissions(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignPermissions(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") t.SetupVartypes() mkline := t.NewMkLine("options.mk", 2, "PKG_DEVELOPER?=\tyes") - MkLineChecker{mkline}.checkVarassignDefPermissions() + MkLineChecker{mkline}.checkVarassignPermissions() t.CheckOutputLines( "WARN: options.mk:2: The variable PKG_DEVELOPER may not be given a default value by any package.") @@ -187,9 +175,7 @@ func (s *Suite) Test_MkLineChecker_Check "COMMENT=\t${GAMES_USER}", "COMMENT:=\t${PKGBASE}", "PYPKGPREFIX=${PKGBASE}") - G.Pkgsrc.UserDefinedVars = map[string]MkLine{ - "GAMES_USER": mklines.mklines[0], - } + G.Pkgsrc.UserDefinedVars.Define("GAMES_USER", mklines.mklines[0]) mklines.Check() @@ -396,9 +382,9 @@ func (s *Suite) Test_MkLineChecker_Check "WARN: ~/options.mk:4: The variable PATH should be quoted as part of a shell word.") } -// The ${VARNAME:=suffix} should only be used with lists. +// The ${VARNAME:=suffix} expression should only be used with lists. // It typically appears in MASTER_SITE definitions. -func (s *Suite) Test_MkLineChecker_CheckVaruse_eq_nonlist(c *check.C) { +func (s *Suite) Test_MkLineChecker_CheckVaruse__eq_nonlist(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -414,3 +400,118 @@ func (s *Suite) Test_MkLineChecker_Check t.CheckOutputLines( "WARN: ~/options.mk:2: The :from=to modifier should only be used with lists.") } + +func (s *Suite) Test_MkLineChecker_CheckVaruse__for(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + t.SetupVartypes() + t.SetupMasterSite("MASTER_SITE_GITHUB", "https://github.com/") + mklines := t.SetupFileMkLines("options.mk", + MkRcsID, + ".for var in a b c", + "\t: ${var}", + ".endfor") + + mklines.Check() + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_CheckVaruse__build_defs(c *check.C) { + t := s.Init(c) + + // XXX: This paragraph should not be necessary since VARBASE and X11_TYPE + // are also defined in vardefs.go. + t.SetupPkgsrc() + t.CreateFileLines("mk/defaults/mk.conf", + "VARBASE?= /usr/pkg/var") + G.Pkgsrc.LoadInfrastructure() + + t.SetupCommandLine("-Wall,no-space") + t.SetupVartypes() + mklines := t.SetupFileMkLines("options.mk", + MkRcsID, + "COMMENT= ${VARBASE} ${X11_TYPE}", + "BUILD_DEFS+= X11_TYPE") + + mklines.Check() + + t.CheckOutputLines( + "WARN: ~/options.mk:2: The user-defined variable VARBASE is used but not added to BUILD_DEFS.") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignSpecific(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + G.Pkgsrc.LoadInfrastructure() + + t.SetupCommandLine("-Wall,no-space") + t.SetupVartypes() + mklines := t.SetupFileMkLines("module.mk", + MkRcsID, + "EGDIR= ${PREFIX}/etc/rc.d", + "_TOOLS_VARNAME.sed= SED", + "DIST_SUBDIR= ${PKGNAME}", + "WRKSRC= ${PKGNAME}", + "SITES_distfile.tar.gz= ${MASTER_SITES_GITHUB:=user/}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: ~/module.mk:2: Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.", + "WARN: ~/module.mk:3: _TOOLS_VARNAME.sed is defined but not used.", + "WARN: ~/module.mk:3: Variable names starting with an underscore (_TOOLS_VARNAME.sed) are reserved for internal pkgsrc use.", + "WARN: ~/module.mk:4: PKGNAME should not be used in DIST_SUBDIR, as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", + "WARN: ~/module.mk:5: PKGNAME should not be used in WRKSRC, as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", + "WARN: ~/module.mk:6: SITES_distfile.tar.gz is defined but not used.", + "WARN: ~/module.mk:6: SITES_* is deprecated. Please use SITES.* instead.") +} + +func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + G.Pkgsrc.LoadInfrastructure() + + t.SetupCommandLine("-Wall,no-space") + mklines := t.SetupFileMkLines("module.mk", + MkRcsID, + "CFLAGS+= -Wl,--rpath,${PREFIX}/lib", + "PKG_FAIL_REASON+= \"Group ${GAMEGRP} doesn't exist.\"") + + mklines.Check() + + t.CheckOutputLines( + "WARN: ~/module.mk:2: Please use ${COMPILER_RPATH_FLAG} instead of \"-Wl,--rpath,\".", + "WARN: ~/module.mk:3: Use of \"GAMEGRP\" is deprecated. Use GAMES_GROUP instead.") +} + +func (s *Suite) Test_MkLineChecker_CheckRelativePath(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + G.Pkgsrc.LoadInfrastructure() + t.CreateFileLines("wip/package/Makefile") + t.CreateFileLines("wip/package/module.mk") + mklines := t.SetupFileMkLines("category/package/module.mk", + MkRcsID, + "DEPENDS+= wip-package-[0-9]*:../../wip/package", + ".include \"../../wip/package/module.mk\"", + "", + "DEPENDS+= unresolvable-[0-9]*:../../lang/${LATEST_PYTHON}", + ".include \"../../lang/${LATEST_PYTHON}/module.mk\"", + "", + ".include \"module.mk\"", + ".include \"../../category/../category/package/module.mk\"", // Oops + ".include \"../../mk/bsd.prefs.mk\"", + ".include \"../package/module.mk\"") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: ~/category/package/module.mk:2: A main pkgsrc package must not depend on a pkgsrc-wip package.", + "ERROR: ~/category/package/module.mk:3: A main pkgsrc package must not depend on a pkgsrc-wip package.", + "WARN: ~/category/package/module.mk:11: Invalid relative path \"../package/module.mk\".") +} Index: pkgsrc/pkgtools/pkglint/files/mkparser_test.go diff -u pkgsrc/pkgtools/pkglint/files/mkparser_test.go:1.13 pkgsrc/pkgtools/pkglint/files/mkparser_test.go:1.14 --- pkgsrc/pkgtools/pkglint/files/mkparser_test.go:1.13 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/mkparser_test.go Wed Sep 5 17:56:22 2018 @@ -1,7 +1,7 @@ package main import ( - check "gopkg.in/check.v1" + "gopkg.in/check.v1" ) func (s *Suite) Test_MkParser_MkTokens(c *check.C) { @@ -162,6 +162,8 @@ func (s *Suite) Test_MkParser_MkCond(c * &mkCond{Not: &mkCond{Empty: varuse("VARNAME")}}) check("!empty(VARNAME:M[yY][eE][sS])", &mkCond{Not: &mkCond{Empty: varuse("VARNAME", "M[yY][eE][sS]")}}) + check("!empty(USE_TOOLS:Mautoconf\\:run)", + &mkCond{Not: &mkCond{Empty: varuse("USE_TOOLS", "Mautoconf:run")}}) check("${VARNAME} != \"Value\"", &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) check("${VARNAME:Mi386} != \"Value\"", Index: pkgsrc/pkgtools/pkglint/files/substcontext_test.go diff -u pkgsrc/pkgtools/pkglint/files/substcontext_test.go:1.13 pkgsrc/pkgtools/pkglint/files/substcontext_test.go:1.14 --- pkgsrc/pkgtools/pkglint/files/substcontext_test.go:1.13 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/substcontext_test.go Wed Sep 5 17:56:22 2018 @@ -90,6 +90,49 @@ func (s *Suite) Test_SubstContext__no_cl "WARN: Makefile:13: Incomplete SUBST block: SUBST_STAGE.repl missing.") } +func (s *Suite) Test_SubstContext__multiple_classes_in_one_line(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wextra") + + simulateSubstLines(t, + "10: SUBST_CLASSES+= one two", + "11: SUBST_STAGE.one= post-configure", + "12: SUBST_FILES.one= one.txt", + "13: SUBST_SED.one= s,one,1,g", + "14: SUBST_STAGE.two= post-configure", + "15: SUBST_FILES.two= two.txt", + "17: ") + + t.CheckOutputLines( + "WARN: Makefile:10: Please add only one class at a time to SUBST_CLASSES.", + "WARN: Makefile:17: Incomplete SUBST block: SUBST_SED.two, SUBST_VARS.two or SUBST_FILTER_CMD.two missing.") +} + +func (s *Suite) Test_SubstContext__multiple_classes_in_one_block(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wextra") + + simulateSubstLines(t, + "10: SUBST_CLASSES+= one", + "11: SUBST_STAGE.one= post-configure", + "12: SUBST_STAGE.one= post-configure", + "13: SUBST_FILES.one= one.txt", + "14: SUBST_CLASSES+= two", // The block "one" is not finished yet. + "15: SUBST_SED.one= s,one,1,g", + "16: SUBST_STAGE.two= post-configure", + "17: SUBST_FILES.two= two.txt", + "18: SUBST_SED.two= s,two,2,g", + "19: ") + + t.CheckOutputLines( + "WARN: Makefile:12: Duplicate definition of \"SUBST_STAGE.one\".", + "WARN: Makefile:14: Incomplete SUBST block: SUBST_SED.one, SUBST_VARS.one or SUBST_FILTER_CMD.one missing.", + "WARN: Makefile:14: Subst block \"one\" should be finished before adding the next class to SUBST_CLASSES.", + "WARN: Makefile:15: Variable \"SUBST_SED.one\" does not match SUBST class \"two\".") +} + func (s *Suite) Test_SubstContext__directives(c *check.C) { t := s.Init(c) @@ -194,6 +237,43 @@ func (s *Suite) Test_SubstContext__post_ "AUTOFIX: os.mk:4: Replacing \"post-patch\" with \"pre-configure\".") } +func (s *Suite) Test_SubstContext__pre_configure_with_NO_CONFIGURE(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + t.SetupPkgsrc() + G.Pkgsrc.LoadInfrastructure() + t.CreateFileLines("category/Makefile") + t.CreateFileLines("licenses/2-clause-bsd") + + t.Chdir("category/package") + t.CreateFileLines("PLIST", + PlistRcsID, + "bin/program") + t.CreateFileLines("Makefile", + MkRcsID, + "", + "CATEGORIES= category", + "", + "COMMENT= Comment", + "LICENSE= 2-clause-bsd", + "", + "SUBST_CLASSES+= os", + "SUBST_STAGE.os= pre-configure", + "SUBST_FILES.os= guess-os.h", + "SUBST_SED.os= -e s,@OPSYS@,Darwin,", + "", + "NO_CHECKSUM= yes", + "NO_CONFIGURE= yes", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + G.checkdirPackage(".") + + t.CheckOutputLines( + "WARN: Makefile:9: SUBST_STAGE pre-configure has no effect when NO_CONFIGURE is set (in line 14).") +} + func (s *Suite) Test_SubstContext__adjacent(c *check.C) { t := s.Init(c) Index: pkgsrc/pkgtools/pkglint/files/licenses_test.go diff -u pkgsrc/pkgtools/pkglint/files/licenses_test.go:1.14 pkgsrc/pkgtools/pkglint/files/licenses_test.go:1.15 --- pkgsrc/pkgtools/pkglint/files/licenses_test.go:1.14 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/licenses_test.go Wed Sep 5 17:56:22 2018 @@ -78,8 +78,9 @@ func (s *Suite) Test_checkToplevelUnused G.Main("pkglint", "-r", "-Cglobal", t.File(".")) t.CheckOutputLines( + "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", // Added by Tester.SetupPkgsrc "WARN: ~/licenses/gnu-gpl-v3: This license seems to be unused.", - "0 errors and 1 warning found.") + "0 errors and 2 warnings found.") } func (s *Suite) Test_LicenseChecker_checkLicenseName__LICENSE_FILE(c *check.C) { Index: pkgsrc/pkgtools/pkglint/files/linechecker_test.go diff -u pkgsrc/pkgtools/pkglint/files/linechecker_test.go:1.6 pkgsrc/pkgtools/pkglint/files/linechecker_test.go:1.7 --- pkgsrc/pkgtools/pkglint/files/linechecker_test.go:1.6 Sat Jan 27 18:50:36 2018 +++ pkgsrc/pkgtools/pkglint/files/linechecker_test.go Wed Sep 5 17:56:22 2018 @@ -11,6 +11,12 @@ func (s *Suite) Test_LineChecker_CheckAb CheckLineAbsolutePathname(line, "bindir=/bin") CheckLineAbsolutePathname(line, "bindir=/../lib") + CheckLineAbsolutePathname(line, "cat /dev/null") // FIXME: Not classified as absolute path. + CheckLineAbsolutePathname(line, "cat /dev//tty") // FIXME: Not classified as absolute patFIXMEh. + CheckLineAbsolutePathname(line, "cat /dev/zero") // FIXME: Not classified as absolute path. + CheckLineAbsolutePathname(line, "cat /dev/stdin") // FIXME: Not classified as absolute path. + CheckLineAbsolutePathname(line, "cat /dev/stdout") // FIXME: Not classified as absolute path. + CheckLineAbsolutePathname(line, "cat /dev/stderr") // FIXME: Not classified as absolute path. t.CheckOutputLines( "WARN: Makefile:1: Found absolute pathname: /bin") Index: pkgsrc/pkgtools/pkglint/files/mkshparser.go diff -u pkgsrc/pkgtools/pkglint/files/mkshparser.go:1.6 pkgsrc/pkgtools/pkglint/files/mkshparser.go:1.7 --- pkgsrc/pkgtools/pkglint/files/mkshparser.go:1.6 Mon Jan 1 18:04:15 2018 +++ pkgsrc/pkgtools/pkglint/files/mkshparser.go Wed Sep 5 17:56:22 2018 @@ -28,7 +28,7 @@ type ParseError struct { } func (e *ParseError) Error() string { - return fmt.Sprintf("parse error at %v", e.RemainingTokens) + return fmt.Sprintf("parse error at %#v", e.RemainingTokens) } type ShellLexer struct { Index: pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go diff -u pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go:1.6 pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go:1.7 --- pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go:1.6 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go Wed Sep 5 17:56:22 2018 @@ -101,22 +101,22 @@ func (s *Suite) Test_Pkgsrc_loadTools(c t.DisableTracing() t.CheckOutputLines( - "TRACE: + (*ToolRegistry).Trace()", - "TRACE: 1 tool &{Name:bzcat Varname: MustUseVarForm:false Predefined:false UsableAtLoadTime:false}", - "TRACE: 1 tool &{Name:bzip2 Varname: MustUseVarForm:false Predefined:false UsableAtLoadTime:false}", - "TRACE: 1 tool &{Name:chown Varname:CHOWN MustUseVarForm:false Predefined:false UsableAtLoadTime:false}", - "TRACE: 1 tool &{Name:echo Varname:ECHO MustUseVarForm:true Predefined:true UsableAtLoadTime:true}", - "TRACE: 1 tool &{Name:echo -n Varname:ECHO_N MustUseVarForm:true Predefined:true UsableAtLoadTime:true}", - "TRACE: 1 tool &{Name:false Varname:FALSE MustUseVarForm:true Predefined:true UsableAtLoadTime:false}", - "TRACE: 1 tool &{Name:gawk Varname:AWK MustUseVarForm:false Predefined:false UsableAtLoadTime:false}", - "TRACE: 1 tool &{Name:m4 Varname: MustUseVarForm:false Predefined:true UsableAtLoadTime:true}", - "TRACE: 1 tool &{Name:msgfmt Varname: MustUseVarForm:false Predefined:false UsableAtLoadTime:false}", - "TRACE: 1 tool &{Name:mv Varname:MV MustUseVarForm:false Predefined:true UsableAtLoadTime:false}", - "TRACE: 1 tool &{Name:pwd Varname:PWD MustUseVarForm:false Predefined:true UsableAtLoadTime:true}", - "TRACE: 1 tool &{Name:strip Varname: MustUseVarForm:false Predefined:false UsableAtLoadTime:false}", - "TRACE: 1 tool &{Name:test Varname:TEST MustUseVarForm:true Predefined:true UsableAtLoadTime:true}", - "TRACE: 1 tool &{Name:true Varname:TRUE MustUseVarForm:true Predefined:true UsableAtLoadTime:true}", - "TRACE: - (*ToolRegistry).Trace()") + "TRACE: + (*Tools).Trace(\"Pkgsrc\")", + "TRACE: 1 tool &{Name:bzcat Varname: MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:bzip2 Varname: MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:chown Varname:CHOWN MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:echo Varname:ECHO MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:echo -n Varname:ECHO_N MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:false Varname:FALSE MustUseVarForm:true Validity:Nowhere}", + "TRACE: 1 tool &{Name:gawk Varname:AWK MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:m4 Varname: MustUseVarForm:false Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:msgfmt Varname: MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:mv Varname:MV MustUseVarForm:false Validity:AtRunTime}", + "TRACE: 1 tool &{Name:pwd Varname:PWD MustUseVarForm:false Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:strip Varname: MustUseVarForm:false Validity:Nowhere}", + "TRACE: 1 tool &{Name:test Varname:TEST MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: 1 tool &{Name:true Varname:TRUE MustUseVarForm:true Validity:AfterPrefsMk}", + "TRACE: - (*Tools).Trace(\"Pkgsrc\")") } func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile(c *check.C) { @@ -158,3 +158,96 @@ func (s *Suite) Test_Pkgsrc_deprecated(c t.CheckOutputLines( "WARN: Makefile:5: Definition of USE_PERL5 is deprecated. Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.") } + +func (s *Suite) Test_Pkgsrc_Latest_no_basedir(c *check.C) { + t := s.Init(c) + + latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") + + c.Check(latest, equals, "") + t.CheckOutputLines( + "ERROR: Cannot find latest version of \"^python[0-9]+$\" in \"~/lang\".") +} + +func (s *Suite) Test_Pkgsrc_Latest_no_subdirs(c *check.C) { + t := s.Init(c) + + t.SetupFileLines("lang/Makefile") + + latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") + + c.Check(latest, equals, "") + t.CheckOutputLines( + "ERROR: Cannot find latest version of \"^python[0-9]+$\" in \"~/lang\".") +} + +func (s *Suite) Test_Pkgsrc_Latest_single(c *check.C) { + t := s.Init(c) + + t.SetupFileLines("lang/Makefile") + t.SetupFileLines("lang/python27/Makefile") + + latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") + + c.Check(latest, equals, "../../lang/python27") +} + +func (s *Suite) Test_Pkgsrc_Latest_multi(c *check.C) { + t := s.Init(c) + + t.SetupFileLines("lang/Makefile") + t.SetupFileLines("lang/python27/Makefile") + t.SetupFileLines("lang/python35/Makefile") + + latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") + + c.Check(latest, equals, "../../lang/python35") +} + +func (s *Suite) Test_Pkgsrc_Latest_numeric(c *check.C) { + t := s.Init(c) + + t.SetupFileLines("databases/postgresql95/Makefile") + t.SetupFileLines("databases/postgresql97/Makefile") + t.SetupFileLines("databases/postgresql100/Makefile") + t.SetupFileLines("databases/postgresql104/Makefile") + + latest := G.Pkgsrc.Latest("databases", `^postgresql[0-9]+$`, "$0") + + c.Check(latest, equals, "postgresql104") +} + +func (s *Suite) Test_Pkgsrc_loadPkgOptions(c *check.C) { + t := s.Init(c) + + t.SetupFileLines("mk/defaults/options.description", + "option-name Description of the option", + "<<<<< Merge conflict", + "===== Merge conflict", + ">>>>> Merge conflict") + + t.ExpectFatal( + G.Pkgsrc.loadPkgOptions, + "FATAL: ~/mk/defaults/options.description:2: Unknown line format.") +} + +func (s *Suite) Test_Pkgsrc_loadTools__no_tools_found(c *check.C) { + t := s.Init(c) + + t.ExpectFatal( + G.Pkgsrc.loadTools, + "FATAL: ~/mk/tools/bsd.tools.mk: Cannot be read.") + + t.CreateFileLines("mk/tools/bsd.tools.mk") + + t.ExpectFatal( + G.Pkgsrc.loadTools, + "FATAL: ~/mk/tools/bsd.tools.mk: Must not be empty.") + + t.CreateFileLines("mk/tools/bsd.tools.mk", + MkRcsID) + + t.ExpectFatal( + G.Pkgsrc.loadTools, + "FATAL: ~/mk/tools/bsd.tools.mk: Too few tool files.") +} Index: pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go diff -u pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go:1.6 pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go:1.7 --- pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go:1.6 Sat Jan 27 18:50:36 2018 +++ pkgsrc/pkgtools/pkglint/files/shtokenizer_test.go Wed Sep 5 17:56:22 2018 @@ -222,10 +222,24 @@ func (s *Suite) Test_ShTokenizer_ShAtom( dquot("s,\\$$sysconfdir/jabberd,\\$$sysconfdir,g"), word("\"")) - check("echo $$,$$/", + check("echo $$, $$- $$/ $$; $$| $$,$$/$$;$$-", word("echo"), space, - word("$$,$$/")) + word("$$,"), + space, + word("$$-"), + space, + word("$$/"), + space, + word("$$"), + semicolon, + space, + word("$$"), + pipe, + space, + word("$$,$$/$$"), + semicolon, + word("$$-")) rest = checkRest("COMMENT=\t\\Make $$$$ fast\"", word("COMMENT="), Index: pkgsrc/pkgtools/pkglint/files/vartype_test.go diff -u pkgsrc/pkgtools/pkglint/files/vartype_test.go:1.6 pkgsrc/pkgtools/pkglint/files/vartype_test.go:1.7 --- pkgsrc/pkgtools/pkglint/files/vartype_test.go:1.6 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/vartype_test.go Wed Sep 5 17:56:22 2018 @@ -1,7 +1,7 @@ package main import ( - check "gopkg.in/check.v1" + "gopkg.in/check.v1" ) func (s *Suite) Test_Vartype_EffectivePermissions(c *check.C) { @@ -44,3 +44,14 @@ func (s *Suite) Test_AclPermissions_Stri c.Check(aclpAll.String(), equals, "set, set-default, append, use-loadtime, use") c.Check(aclpUnknown.String(), equals, "unknown") } + +func (s *Suite) Test_Vartype_IsConsideredList(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + + c.Check(G.Pkgsrc.VariableType("COMMENT").IsConsideredList(), equals, false) + c.Check(G.Pkgsrc.VariableType("DEPENDS").IsConsideredList(), equals, false) + c.Check(G.Pkgsrc.VariableType("PKG_FAIL_REASON").IsConsideredList(), equals, true) + c.Check(G.Pkgsrc.VariableType("CONF_FILES").IsConsideredList(), equals, true) +} Index: pkgsrc/pkgtools/pkglint/files/mkline.go diff -u pkgsrc/pkgtools/pkglint/files/mkline.go:1.36 pkgsrc/pkgtools/pkglint/files/mkline.go:1.37 --- pkgsrc/pkgtools/pkglint/files/mkline.go:1.36 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/mkline.go Wed Sep 5 17:56:22 2018 @@ -138,7 +138,7 @@ func NewMkLine(line Line) *MkLineImpl { return &MkLineImpl{line, nil} } - line.Errorf("Unknown Makefile line format.") + line.Errorf("Unknown Makefile line format: %q.", text) return &MkLineImpl{line, nil} } @@ -278,14 +278,14 @@ func (mkline *MkLineImpl) SetConditional // Example: // input: ${PREFIX}/bin abc // output: [MkToken("${PREFIX}", MkVarUse("PREFIX")), MkToken("/bin abc")] -func (mkline *MkLineImpl) Tokenize(s string) []*MkToken { +func (mkline *MkLineImpl) Tokenize(s string, warn bool) []*MkToken { if trace.Tracing { defer trace.Call(mkline, s)() } p := NewMkParser(mkline.Line, s, true) tokens := p.MkTokens() - if p.Rest() != "" { + if warn && p.Rest() != "" { mkline.Warnf("Pkglint parse error in MkLine.Tokenize at %q.", p.Rest()) } return tokens @@ -298,7 +298,7 @@ func (mkline *MkLineImpl) Tokenize(s str // // If the separator is empty, splitting is done on whitespace. func (mkline *MkLineImpl) ValueSplit(value string, separator string) []string { - tokens := mkline.Tokenize(value) + tokens := mkline.Tokenize(value, false) var split []string for _, token := range tokens { if split == nil { @@ -320,6 +320,10 @@ func (mkline *MkLineImpl) ValueSplit(val return split } +func (mkline *MkLineImpl) ValueTokens() []*MkToken { + return mkline.Tokenize(mkline.Value(), false) +} + func (mkline *MkLineImpl) WithoutMakeVariables(value string) string { valueNovar := value for { @@ -477,7 +481,7 @@ func (nq NeedsQuoting) String() string { func (mkline *MkLineImpl) VariableNeedsQuoting(varname string, vartype *Vartype, vuc *VarUseContext) (needsQuoting NeedsQuoting) { if trace.Tracing { - defer trace.Call(varname, vartype, vuc, "=>", &needsQuoting)() + defer trace.Call(varname, vartype, vuc, trace.Result(&needsQuoting))() } if vartype == nil || vuc.vartype == nil { @@ -528,7 +532,7 @@ func (mkline *MkLineImpl) VariableNeedsQ // Pkglint assumes that the tool definitions don't include very // special characters, so they can safely be used inside any quotes. - if G.Pkgsrc.Tools.ByVarname(varname) != nil { + if tool := G.ToolByVarname(varname, vuc.time.ToToolTime()); tool != nil { switch vuc.quoting { case vucQuotPlain: if !vuc.IsWordPart { @@ -578,85 +582,6 @@ func (mkline *MkLineImpl) VariableNeedsQ return nqDontKnow } -// Returns the type of the variable (possibly guessed based on the variable name), -// or nil if the type cannot even be guessed. -func (mkline *MkLineImpl) VariableType(varname string) *Vartype { - if trace.Tracing { - defer trace.Call1(varname)() - } - - if vartype := G.Pkgsrc.vartypes[varname]; vartype != nil { - return vartype - } - if vartype := G.Pkgsrc.vartypes[varnameCanon(varname)]; vartype != nil { - return vartype - } - - if tool := G.Pkgsrc.Tools.ByVarname(varname); tool != nil { - perms := aclpUse - if trace.Tracing { - trace.Stepf("Use of tool %+v", tool) - } - if tool.UsableAtLoadTime { - if G.Pkg == nil || G.Pkg.SeenBsdPrefsMk || G.Pkg.loadTimeTools[tool.Name] { - perms |= aclpUseLoadtime - } - } - return &Vartype{lkNone, BtShellCommand, []ACLEntry{{"*", perms}}, false} - } - - m, toolvarname := match1(varname, `^TOOLS_(.*)`) - if m && G.Pkgsrc.Tools.ByVarname(toolvarname) != nil { - return &Vartype{lkNone, BtPathname, []ACLEntry{{"*", aclpUse}}, false} - } - - allowAll := []ACLEntry{{"*", aclpAll}} - allowRuntime := []ACLEntry{{"*", aclpAllRuntime}} - - // Guess the data type of the variable based on naming conventions. - varbase := varnameBase(varname) - var gtype *Vartype - switch { - case hasSuffix(varbase, "DIRS"): - gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} - case hasSuffix(varbase, "DIR") && !hasSuffix(varbase, "DESTDIR"), hasSuffix(varname, "_HOME"): - gtype = &Vartype{lkNone, BtPathname, allowRuntime, true} - case hasSuffix(varbase, "FILES"): - gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} - case hasSuffix(varbase, "FILE"): - gtype = &Vartype{lkNone, BtPathname, allowRuntime, true} - case hasSuffix(varbase, "PATH"): - gtype = &Vartype{lkNone, BtPathlist, allowRuntime, true} - case hasSuffix(varbase, "PATHS"): - gtype = &Vartype{lkShell, BtPathname, allowRuntime, true} - case hasSuffix(varbase, "_USER"): - gtype = &Vartype{lkNone, BtUserGroupName, allowAll, true} - case hasSuffix(varbase, "_GROUP"): - gtype = &Vartype{lkNone, BtUserGroupName, allowAll, true} - case hasSuffix(varbase, "_ENV"): - gtype = &Vartype{lkShell, BtShellWord, allowRuntime, true} - case hasSuffix(varbase, "_CMD"): - gtype = &Vartype{lkNone, BtShellCommand, allowRuntime, true} - case hasSuffix(varbase, "_ARGS"): - gtype = &Vartype{lkShell, BtShellWord, allowRuntime, true} - case hasSuffix(varbase, "_CFLAGS"), hasSuffix(varname, "_CPPFLAGS"), hasSuffix(varname, "_CXXFLAGS"): - gtype = &Vartype{lkShell, BtCFlag, allowRuntime, true} - case hasSuffix(varname, "_LDFLAGS"): - gtype = &Vartype{lkShell, BtLdFlag, allowRuntime, true} - case hasSuffix(varbase, "_MK"): - gtype = &Vartype{lkNone, BtUnknown, allowAll, true} - } - - if trace.Tracing { - if gtype != nil { - trace.Step2("The guessed type of %q is %q.", varname, gtype.String()) - } else { - trace.Step1("No type definition found for %q.", varname) - } - } - return gtype -} - // TODO: merge with determineUsedVariables func (mkline *MkLineImpl) ExtractUsedVariables(text string) []string { re := regex.Compile(`^(?:[^\$]+|\$[\$*<>?@]|\$\{([.0-9A-Z_a-z]+)(?::(?:[^\${}]|\$[^{])+)?\})`) @@ -720,6 +645,38 @@ func (mkline *MkLineImpl) DetermineUsedV } } +type MkOperator uint8 + +const ( + opAssign MkOperator = iota // = + opAssignShell // != + opAssignEval // := + opAssignAppend // += + opAssignDefault // ?= + opUseCompare // A variable is compared to a value, e.g. in a condition. + opUseMatch // A variable is matched using the :M or :N modifier. +) + +func NewMkOperator(op string) MkOperator { + switch op { + case "=": + return opAssign + case "!=": + return opAssignShell + case ":=": + return opAssignEval + case "+=": + return opAssignAppend + case "?=": + return opAssignDefault + } + panic("Invalid operator: " + op) +} + +func (op MkOperator) String() string { + return [...]string{"=", "!=", ":=", "+=", "?=", "use", "use-loadtime", "use-match"}[op] +} + // VarUseContext defines the context in which a variable is defined // or used. Whether that is allowed depends on: // @@ -759,6 +716,13 @@ const ( func (t vucTime) String() string { return [...]string{"unknown", "parse", "run"}[t] } +func (t vucTime) ToToolTime() ToolTime { + if t == vucTimeParse { + return LoadTime + } + return RunTime +} + // The quoting context in which the variable is used. // Depending on this context, the modifiers :Q or :M can be allowed or not. type vucQuoting uint8 Index: pkgsrc/pkgtools/pkglint/files/pkglint.go diff -u pkgsrc/pkgtools/pkglint/files/pkglint.go:1.36 pkgsrc/pkgtools/pkglint/files/pkglint.go:1.37 --- pkgsrc/pkgtools/pkglint/files/pkglint.go:1.36 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/pkglint.go Wed Sep 5 17:56:22 2018 @@ -117,6 +117,10 @@ func main() { // Main runs the main program with the given arguments. // argv[0] is the program name. +// +// Note: during tests, calling this method disables tracing +// because the command line option --debug sets trace.Tracing +// back to false. func (pkglint *Pkglint) Main(argv ...string) (exitcode int) { defer func() { if r := recover(); r != nil { @@ -137,11 +141,10 @@ func (pkglint *Pkglint) Main(argv ...str if err != nil { dummyLine.Fatalf("Cannot create profiling file: %s", err) } + defer f.Close() + pprof.StartCPUProfile(f) - defer func() { - pprof.StopCPUProfile() - f.Close() - }() + defer pprof.StopCPUProfile() regex.Profiling = true pkglint.loghisto = histogram.New() @@ -150,7 +153,7 @@ func (pkglint *Pkglint) Main(argv ...str pkglint.logOut.Write("") pkglint.loghisto.PrintStats("loghisto", pkglint.logOut.out, -1) regex.PrintStats(pkglint.logOut.out) - pkglint.loaded.PrintStats("loaded", pkglint.logOut.out, 50) + pkglint.loaded.PrintStats("loaded", pkglint.logOut.out, 10) }() } @@ -300,9 +303,9 @@ func (pkglint *Pkglint) CheckDirent(fnam isReg := st.Mode().IsRegular() dir := ifelseStr(isReg, path.Dir(fname), fname) - absCurrentDir := abspath(dir) - pkglint.Wip = !pkglint.opts.Import && matches(absCurrentDir, `/wip/|/wip$`) - pkglint.Infrastructure = matches(absCurrentDir, `/mk/|/mk$`) + pkgsrcRel := G.Pkgsrc.ToRel(dir) + pkglint.Wip = matches(pkgsrcRel, `^wip(/|$)`) + pkglint.Infrastructure = matches(pkgsrcRel, `^mk(/|$)`) pkgsrcdir := findPkgsrcTopdir(dir) if pkgsrcdir == "" { NewLineWhole(fname).Errorf("Cannot determine the pkgsrc root directory for %q.", cleanpath(dir)) @@ -346,25 +349,33 @@ func resolveVariableRefs(text string) st visited := make(map[string]bool) // To prevent endless loops - str := text - for { - replaced := regex.Compile(`\$\{([\w.]+)\}`).ReplaceAllStringFunc(str, func(m string) string { - varname := m[2 : len(m)-1] - if !visited[varname] { - visited[varname] = true - if G.Pkg != nil { - if value, ok := G.Pkg.varValue(varname); ok { - return value - } + replacer := func(m string) string { + varname := m[2 : len(m)-1] + if !visited[varname] { + visited[varname] = true + if G.Pkg != nil { + switch varname { + case "KRB5_TYPE": + return "heimdal" + case "PGSQL_VERSION": + return "95" } - if G.Mk != nil { - if value, ok := G.Mk.VarValue(varname); ok { - return value - } + if mkline := G.Pkg.vars.FirstDefinition(varname); mkline != nil { + return mkline.Value() } } - return "${" + varname + "}" - }) + if G.Mk != nil { + if value, ok := G.Mk.vars.Value(varname); ok { + return value + } + } + } + return "${" + varname + "}" + } + + str := text + for { + replaced := regex.Compile(`\$\{([\w.]+)\}`).ReplaceAllStringFunc(str, replacer) if replaced == str { return replaced } @@ -476,7 +487,23 @@ func (pkglint *Pkglint) Checkfile(fname } basename := path.Base(fname) - if hasPrefix(basename, "work") || hasSuffix(basename, "~") || hasSuffix(basename, ".orig") || hasSuffix(basename, ".rej") { + pkgsrcRel := G.Pkgsrc.ToRel(fname) + depth := strings.Count(pkgsrcRel, "/") + + if depth == 2 && !G.Wip { + if contains(basename, "README") || contains(basename, "TODO") { + NewLineWhole(fname).Errorf("Packages in main pkgsrc must not have a %s file.", basename) + return + } + } + + switch { + case hasPrefix(basename, "work"), + hasSuffix(basename, "~"), + hasSuffix(basename, ".orig"), + hasSuffix(basename, ".rej"), + contains(basename, "README") && depth == 2, + contains(basename, "TODO") && depth == 2: if pkglint.opts.Import { NewLineWhole(fname).Errorf("Must be cleaned up before committing the package.") } @@ -485,7 +512,7 @@ func (pkglint *Pkglint) Checkfile(fname st, err := os.Lstat(fname) if err != nil { - NewLineWhole(fname).Errorf("%s", err) + NewLineWhole(fname).Errorf("Cannot determine file type: %s", err) return } @@ -635,3 +662,80 @@ func ChecklinesTrailingEmptyLines(lines lines[last].Notef("Trailing empty lines.") } } + +// Tool returns the tool definition from the closest scope (file, global), or nil. +// The command can be "sed" or "gsed" or "${SED}". +// If a tool is returned, usable tells whether that tool has been added +// to USE_TOOLS in the current scope. +func (pkglint *Pkglint) Tool(command string, time ToolTime) (tool *Tool, usable bool) { + varname := "" + if m, toolVarname := match1(command, `^\$\{(\w+)\}$`); m { + varname = toolVarname + } + + if G.Mk != nil { + tools := G.Mk.Tools + if t := tools.ByName(command); t != nil { + if tools.Usable(t, time) { + return t, true + } + tool = t + } + + if t := tools.ByVarname(varname); t != nil { + if tools.Usable(t, time) { + return t, true + } + if tool == nil { + tool = t + } + } + } + + tools := G.Pkgsrc.Tools + if t := tools.ByName(command); t != nil { + if tools.Usable(t, time) { + return t, true + } + if tool == nil { + tool = t + } + } + + if t := tools.ByVarname(varname); t != nil { + if tools.Usable(t, time) { + return t, true + } + if tool == nil { + tool = t + } + } + + return +} + +func (pkglint *Pkglint) ToolByVarname(varname string, time ToolTime) *Tool { + + var tool *Tool + if G.Mk != nil { + tools := G.Mk.Tools + if t := tools.ByVarname(varname); t != nil { + if tools.Usable(t, time) { + return t + } + tool = t + } + } + + tools := G.Pkgsrc.Tools + if t := tools.ByVarname(varname); t != nil { + if tools.Usable(t, time) { + return t + } + if tool == nil { + tool = t + } + } + + return tool +} Index: pkgsrc/pkgtools/pkglint/files/mkline_test.go diff -u pkgsrc/pkgtools/pkglint/files/mkline_test.go:1.40 pkgsrc/pkgtools/pkglint/files/mkline_test.go:1.41 --- pkgsrc/pkgtools/pkglint/files/mkline_test.go:1.40 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/mkline_test.go Wed Sep 5 17:56:22 2018 @@ -222,15 +222,14 @@ func (s *Suite) Test_NewMkLine__autofix_ func (s *Suite) Test_MkLine_VariableType_varparam(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("fname", 1, "# dummy") t.SetupVartypes() - t1 := mkline.VariableType("FONT_DIRS") + t1 := G.Pkgsrc.VariableType("FONT_DIRS") c.Assert(t1, check.NotNil) c.Check(t1.String(), equals, "ShellList of Pathmask (guessed)") - t2 := mkline.VariableType("FONT_DIRS.ttf") + t2 := G.Pkgsrc.VariableType("FONT_DIRS.ttf") c.Assert(t2, check.NotNil) c.Check(t2.String(), equals, "ShellList of Pathmask (guessed)") @@ -240,8 +239,7 @@ func (s *Suite) Test_VarUseContext_Strin t := s.Init(c) t.SetupVartypes() - mkline := t.NewMkLine("fname", 1, "# dummy") - vartype := mkline.VariableType("PKGNAME") + vartype := G.Pkgsrc.VariableType("PKGNAME") vuc := &VarUseContext{vartype, vucTimeUnknown, vucQuotBackt, false} c.Check(vuc.String(), equals, "(PkgName time:unknown quoting:backt wordpart:false)") @@ -389,8 +387,8 @@ func (s *Suite) Test_MkLine_variableNeed t.SetupCommandLine("-Wall") t.SetupVartypes() - t.SetupTool(&Tool{Name: "find", Varname: "FIND", Predefined: true}) - t.SetupTool(&Tool{Name: "sort", Varname: "SORT", Predefined: true}) + t.SetupToolUsable("find", "FIND") + t.SetupToolUsable("sort", "SORT") G.Pkg = NewPackage(t.File("category/pkgbase")) G.Mk = t.NewMkLines("Makefile", MkRcsID, @@ -427,16 +425,15 @@ func (s *Suite) Test_MkLine_variableNeed t := s.Init(c) t.SetupCommandLine("-Wall") - t.SetupTool(&Tool{Name: "perl", Varname: "PERL5", Predefined: true}) - t.SetupTool(&Tool{Name: "bash", Varname: "BASH", Predefined: true}) + t.SetupToolUsable("perl", "PERL5") + t.SetupToolUsable("bash", "BASH") t.SetupVartypes() - G.Mk = t.NewMkLines("Makefile", + mklines := t.NewMkLines("Makefile", MkRcsID, "\t${RUN} cd ${WRKSRC} && ( ${ECHO} ${PERL5:Q} ; ${ECHO} ) | ${BASH} ./install", "\t${RUN} cd ${WRKSRC} && ( ${ECHO} ${PERL5} ; ${ECHO} ) | ${BASH} ./install") - MkLineChecker{G.Mk.mklines[1]}.Check() - MkLineChecker{G.Mk.mklines[2]}.Check() + mklines.Check() t.CheckOutputLines( "WARN: Makefile:2: The exitcode of the command at the left of the | operator is ignored.", @@ -468,8 +465,8 @@ func (s *Suite) Test_MkLine_variableNeed t.SetupCommandLine("-Wall") t.SetupVartypes() - t.SetupTool(&Tool{Name: "awk", Varname: "AWK", Predefined: true}) - t.SetupTool(&Tool{Name: "echo", Varname: "ECHO", Predefined: true}) + t.SetupToolUsable("awk", "AWK") + t.SetupToolUsable("echo", "ECHO") G.Mk = t.NewMkLines("xpi.mk", MkRcsID, "\t id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"", @@ -549,8 +546,8 @@ func (s *Suite) Test_MkLine_variableNeed t := s.Init(c) t.SetupCommandLine("-Wall") - t.SetupTool(&Tool{Name: "echo", Varname: "ECHO", Predefined: true}) - t.SetupTool(&Tool{Name: "sh", Varname: "SH", Predefined: true}) + t.SetupToolUsable("echo", "ECHO") + t.SetupToolUsable("sh", "SH") t.SetupVartypes() G.Mk = t.NewMkLines("x11/labltk/Makefile", MkRcsID, @@ -598,7 +595,7 @@ func (s *Suite) Test_MkLine_variableNeed t.SetupVartypes() G.Mk = t.NewMkLines("audio/jack-rack/Makefile", MkRcsID, - "LADSPA_PLUGIN_PATH?=\t${PREFIX}/lib/ladspa", + "LADSPA_PLUGIN_PATH=\t${PREFIX}/lib/ladspa", "CPPFLAGS+=\t\t-DLADSPA_PATH=\"\\\"${LADSPA_PLUGIN_PATH}\\\"\"") G.Mk.Check() @@ -641,7 +638,7 @@ func (s *Suite) Test_MkLine_variableNeed t.SetupCommandLine("-Wall") t.SetupVartypes() - G.Pkgsrc.Tools.RegisterVarname("tar", "TAR", dummyMkLine) + t.SetupToolUsable("tar", "TAR") mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -662,7 +659,7 @@ func (s *Suite) Test_MkLine_variableNeed t.SetupCommandLine("-Wall") t.SetupVartypes() - G.Pkgsrc.Tools.RegisterVarname("cat", "CAT", dummyMkLine) + t.SetupToolUsable("cat", "CAT") mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -737,7 +734,7 @@ func (s *Suite) Test_MkLine_variableNeed t.SetupCommandLine("-Wall,no-space") t.SetupVartypes() - t.SetupTool(&Tool{Varname: "BASH"}) + t.SetupToolUsable("bash", "BASH") mklines := t.SetupFileMkLines("Makefile", MkRcsID, @@ -847,7 +844,7 @@ func (s *Suite) Test_MkLine_VariableType t.SetupVartypes() checkType := func(varname string, vartype string) { - actualType := dummyMkLine.VariableType(varname) + actualType := G.Pkgsrc.VariableType(varname) if vartype == "" { c.Check(actualType, check.IsNil) } else { @@ -861,8 +858,8 @@ func (s *Suite) Test_MkLine_VariableType checkType("SOME_DIR", "Pathname (guessed)") checkType("SOMEDIR", "Pathname (guessed)") checkType("SEARCHPATHS", "ShellList of Pathname (guessed)") - checkType("APACHE_USER", "UserGroupName (guessed)") - checkType("APACHE_GROUP", "UserGroupName (guessed)") + checkType("MYPACKAGE_USER", "UserGroupName (guessed)") + checkType("MYPACKAGE_GROUP", "UserGroupName (guessed)") checkType("MY_CMD_ENV", "ShellList of ShellWord (guessed)") checkType("MY_CMD_ARGS", "ShellList of ShellWord (guessed)") checkType("MY_CMD_CFLAGS", "ShellList of CFlag (guessed)") @@ -997,6 +994,13 @@ func (s *Suite) Test_MatchVarassign(c *c checkNotVarassign("# VAR=value") } +func (s *Suite) Test_NewMkOperator(c *check.C) { + c.Check(NewMkOperator(":="), equals, opAssignEval) + c.Check(NewMkOperator("="), equals, opAssign) + + c.Check(func() { NewMkOperator("???") }, check.Panics, "Invalid operator: ???") +} + func (s *Suite) Test_Indentation(c *check.C) { t := s.Init(c) Index: pkgsrc/pkgtools/pkglint/files/mklinechecker.go diff -u pkgsrc/pkgtools/pkglint/files/mklinechecker.go:1.17 pkgsrc/pkgtools/pkglint/files/mklinechecker.go:1.18 --- pkgsrc/pkgtools/pkglint/files/mklinechecker.go:1.17 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/mklinechecker.go Wed Sep 5 17:56:22 2018 @@ -78,18 +78,10 @@ func (ck MkLineChecker) checkInclude() { "that, both this one and the other package should include the", "Makefile.common.") - case includefile == "../../mk/bsd.prefs.mk": - if path.Base(mkline.Filename) == "buildlink3.mk" { + case IsPrefs(includefile): + if path.Base(mkline.Filename) == "buildlink3.mk" && includefile == "../../mk/bsd.prefs.mk" { mkline.Notef("For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") } - if G.Pkg != nil { - G.Pkg.setSeenBsdPrefsMk() - } - - case includefile == "../../mk/bsd.fast.prefs.mk", includefile == "../../mk/buildlink3/bsd.builtin.mk": - if G.Pkg != nil { - G.Pkg.setSeenBsdPrefsMk() - } case hasSuffix(includefile, "/x11-links/buildlink3.mk"): mkline.Errorf("%s must not be included directly. Include \"../../mk/x11.buildlink3.mk\" instead.", includefile) @@ -199,7 +191,7 @@ func (ck MkLineChecker) checkDirectiveFo guessed := true for _, value := range splitOnSpace(values) { if m, vname := match1(value, `^\$\{(.*)\}`); m { - vartype := mkline.VariableType(vname) + vartype := G.Pkgsrc.VariableType(vname) if vartype != nil && !vartype.guessed { guessed = false } @@ -268,7 +260,11 @@ func (ck MkLineChecker) checkDependencyR } } -func (ck MkLineChecker) checkVarassignDefPermissions() { +// checkVarassignPermissions checks the permissions for the left-hand side +// of a variable assignment line. +// +// See checkVarusePermissions. +func (ck MkLineChecker) checkVarassignPermissions() { if !G.opts.WarnPerm || G.Infrastructure { return } @@ -279,7 +275,7 @@ func (ck MkLineChecker) checkVarassignDe mkline := ck.MkLine varname := mkline.Varname() op := mkline.Op() - vartype := mkline.VariableType(varname) + vartype := G.Pkgsrc.VariableType(varname) if vartype == nil { if trace.Tracing { trace.Step1("No type definition found for %q.", varname) @@ -288,6 +284,15 @@ func (ck MkLineChecker) checkVarassignDe } perms := vartype.EffectivePermissions(mkline.Filename) + + // E.g. USE_TOOLS:= ${USE_TOOLS:Nunwanted-tool} + if op == opAssignEval && perms&aclpAppend != 0 { + tokens := mkline.ValueTokens() + if len(tokens) == 1 && tokens[0].Varuse != nil && tokens[0].Varuse.varname == varname { + return + } + } + var needed ACLPermissions switch op { case opAssign, opAssignShell, opAssignEval: @@ -341,13 +346,12 @@ func (ck MkLineChecker) CheckVaruse(varu } varname := varuse.varname - vartype := mkline.VariableType(varname) + vartype := G.Pkgsrc.VariableType(varname) switch { case !G.opts.WarnExtra: case vartype != nil && !vartype.guessed: // Well-known variables are probably defined by the infrastructure. case varIsUsed(varname): - case G.Mk != nil && G.Mk.forVars[varname]: case containsVarRef(varname): default: mkline.Warnf("%s is used but not defined.", varname) @@ -365,7 +369,14 @@ func (ck MkLineChecker) CheckVaruse(varu "This is a much clearer expression of the same thought.") } - ck.CheckVarusePermissions(varname, vartype, vuc) + if varuse.varname == "@" { + ck.MkLine.Warnf("Please use %q instead of %q.", "${.TARGET}", "$@") + Explain( + "It is more readable and prevents confusion with the shell variable", + "of the same name.") + } + + ck.checkVarusePermissions(varname, vartype, vuc) if varname == "LOCALBASE" && !G.Infrastructure { ck.WarnVaruseLocalbase() @@ -381,7 +392,7 @@ func (ck MkLineChecker) CheckVaruse(varu ck.CheckVaruseShellword(varname, vartype, vuc, varuse.Mod(), needsQuoting) } - if G.Pkgsrc.UserDefinedVars[varname] != nil && !G.Pkgsrc.IsBuildDef(varname) && !G.Mk.buildDefs[varname] { + if G.Pkgsrc.UserDefinedVars.Defined(varname) && !G.Pkgsrc.IsBuildDef(varname) && !G.Mk.buildDefs[varname] { mkline.Warnf("The user-defined variable %s is used but not added to BUILD_DEFS.", varname) Explain( "When a pkgsrc package is built, many things can be configured by the", @@ -392,7 +403,11 @@ func (ck MkLineChecker) CheckVaruse(varu } } -func (ck MkLineChecker) CheckVarusePermissions(varname string, vartype *Vartype, vuc *VarUseContext) { +// checkVarusePermissions checks the permissions for the right-hand side +// of a variable assignment line. +// +// See checkVarassignPermissions. +func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, vuc *VarUseContext) { if !G.opts.WarnPerm { return } @@ -432,30 +447,73 @@ func (ck MkLineChecker) CheckVarusePermi isIndirect = true } - done := false - tool := G.Pkgsrc.Tools.ByVarname(varname) - - if isLoadTime && tool != nil { - done = tool.Predefined && (G.Mk == nil || G.Mk.SeenBsdPrefsMk || G.Pkg == nil || G.Pkg.SeenBsdPrefsMk) + if isLoadTime { + if tool := G.ToolByVarname(varname, LoadTime); tool != nil { + ck.checkVaruseToolLoadTime(varname, tool) + } else { + ck.checkVaruseLoadTime(varname, isIndirect) + } + } - if !done && G.Pkg != nil && !G.Pkg.SeenBsdPrefsMk && G.Mk != nil && !G.Mk.SeenBsdPrefsMk { - mkline.Warnf("To use the tool %q at load time, bsd.prefs.mk has to be included before.", varname) - done = true + if !perms.Contains(aclpUseLoadtime) && !perms.Contains(aclpUse) { + needed := aclpUse + if isLoadTime { + needed = aclpUseLoadtime } + alternativeFiles := vartype.AllowedFiles(needed) + if alternativeFiles != "" { + mkline.Warnf("%s may not be used in this file; it would be ok in %s.", + varname, alternativeFiles) + } else { + mkline.Warnf("%s may not be used in any file; it is a write-only variable.", varname) + } + Explain( + "The allowed actions for a variable are determined based on the file", + "name in which the variable is used or defined. The exact rules are", + "hard-coded into pkglint. If they seem to be incorrect, please ask", + "on the tech-pkg@NetBSD.org mailing list.") + } +} - if !done && G.Pkg != nil { - usable, defined := G.Pkg.loadTimeTools[tool.Name] - if usable { - done = true - } - if defined && !usable { - mkline.Warnf("To use the tool %q at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.", varname) - done = true - } +// checkVaruseToolLoadTime checks whether the tool ${varname} may be used at load time. +func (ck MkLineChecker) checkVaruseToolLoadTime(varname string, tool *Tool) { + if tool.UsableAtLoadTime(G.Mk.Tools.SeenPrefs) { + return + } + + if tool.Validity == AfterPrefsMk { + ck.MkLine.Warnf("To use the tool ${%s} at load time, bsd.prefs.mk has to be included before.", varname) + return + } + + if path.Base(ck.MkLine.Filename) == "Makefile" { + pkgsrcTool := G.Pkgsrc.Tools.ByName(tool.Name) + if pkgsrcTool != nil && pkgsrcTool.Validity == Nowhere { + // The tool must have been added too late to USE_TOOLS, + // i.e. after bsd.prefs.mk has been included. + ck.MkLine.Warnf("To use the tool ${%s} at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.", varname) + return } } - if !done && isLoadTime && !isIndirect { + ck.MkLine.Warnf("The tool ${%s} cannot be used at load time.", varname) + Explain( + "To use a tool at load time, it must be declared in the package", + "Makefile by adding it to USE_TOOLS. After that, bsd.prefs.mk must", + "be included. Adding the tool to USE_TOOLS at any later time has", + "no effect, which means that the tool can only be used at run time.", + "That's the rule for the package Makefiles.", + "", + "Since any other .mk file can be included from anywhere else, there", + "is no guarantee that the tool is properly defined for using it at", + "load time (see above for the tricky rules). Therefore the tools can", + "only be used at run time, except in the package Makefile itself.") +} + +func (ck MkLineChecker) checkVaruseLoadTime(varname string, isIndirect bool) { + mkline := ck.MkLine + + if !isIndirect { mkline.Warnf("%s should not be evaluated at load time.", varname) Explain( "Many variables, especially lists of something, get their values", @@ -467,36 +525,16 @@ func (ck MkLineChecker) CheckVarusePermi "Additionally, when using the \":=\" operator, each $$ is replaced", "with a single $, so variables that have references to shell", "variables or regular expressions are modified in a subtle way.") - done = true + return } - if !done && isLoadTime && isIndirect { + if isIndirect { mkline.Warnf("%s should not be evaluated indirectly at load time.", varname) Explain( "The variable on the left-hand side may be evaluated at load time,", "but the variable on the right-hand side may not. Because of the", "assignment in this line, the variable might be used indirectly", "at load time, before it is guaranteed to be properly initialized.") - done = true - } - - if !perms.Contains(aclpUseLoadtime) && !perms.Contains(aclpUse) { - needed := aclpUse - if isLoadTime { - needed = aclpUseLoadtime - } - alternativeFiles := vartype.AllowedFiles(needed) - if alternativeFiles != "" { - mkline.Warnf("%s may not be used in this file; it would be ok in %s.", - varname, alternativeFiles) - } else { - mkline.Warnf("%s may not be used in any file; it is a write-only variable.", varname) - } - Explain( - "The allowed actions for a variable are determined based on the file", - "name in which the variable is used or defined. The exact rules are", - "hard-coded into pkglint. If they seem to be incorrect, please ask", - "on the tech-pkg@NetBSD.org mailing list.") } } @@ -697,7 +735,7 @@ func (ck MkLineChecker) checkVarassign() } defineVar(mkline, varname) - ck.checkVarassignDefPermissions() + ck.checkVarassignPermissions() ck.checkVarassignBsdPrefs() ck.checkText(value) @@ -731,25 +769,6 @@ func (ck MkLineChecker) checkVarassign() } } - if varname == "USE_TOOLS" { - for _, fullToolname := range splitOnSpace(value) { - toolname := strings.Split(fullToolname, ":")[0] - if G.Pkg != nil { - if !G.Pkg.SeenBsdPrefsMk { - G.Pkg.loadTimeTools[toolname] = true - if trace.Tracing { - trace.Step1("loadTimeTool %q", toolname) - } - } else if !G.Pkg.loadTimeTools[toolname] { - G.Pkg.loadTimeTools[toolname] = false - if trace.Tracing { - trace.Step1("too late for loadTimeTool %q", toolname) - } - } - } - } - } - if fix := G.Pkgsrc.Deprecated[varname]; fix != "" { mkline.Warnf("Definition of %s is deprecated. %s", varname, fix) } else if fix := G.Pkgsrc.Deprecated[varcanon]; fix != "" { @@ -773,7 +792,7 @@ func (ck MkLineChecker) checkVarassignVa time = vucTimeParse } - vartype := mkline.VariableType(mkline.Varname()) + vartype := G.Pkgsrc.VariableType(mkline.Varname()) if op == opAssignShell { vartype = shellcommandsContextType } @@ -872,28 +891,35 @@ func (ck MkLineChecker) checkVarassignSp func (ck MkLineChecker) checkVarassignBsdPrefs() { mkline := ck.MkLine - if G.opts.WarnExtra && mkline.Op() == opAssignDefault && G.Pkg != nil && !G.Pkg.SeenBsdPrefsMk { - switch mkline.Varcanon() { - case "BUILDLINK_PKGSRCDIR.*", "BUILDLINK_DEPMETHOD.*", "BUILDLINK_ABI_DEPENDS.*": - return - } - if G.Mk != nil && !G.Mk.FirstTime("include-bsd.prefs.mk") { - return - } + switch mkline.Varcanon() { + case "BUILDLINK_PKGSRCDIR.*", + "BUILDLINK_DEPMETHOD.*", + "BUILDLINK_ABI_DEPENDS.*", + "BUILDLINK_INCDIRS.*", + "BUILDLINK_LIBDIRS.*": + return + } - mkline.Warnf("Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") - Explain( - "The ?= operator is used to provide a default value to a variable.", - "In pkgsrc, many variables can be set by the pkgsrc user in the", - "mk.conf file. This file must be included explicitly. If a ?=", - "operator appears before mk.conf has been included, it will not care", - "about the user's preferences, which can result in unexpected", - "behavior.", - "", - "The easiest way to include the mk.conf file is by including the", - "bsd.prefs.mk file, which will take care of everything.") + if !G.opts.WarnExtra || + G.Infrastructure || + mkline.Op() != opAssignDefault || + G.Mk.Tools.SeenPrefs || + !G.Mk.FirstTime("include bsd.prefs.mk before using ?=") { + return } + + mkline.Warnf("Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") + Explain( + "The ?= operator is used to provide a default value to a variable.", + "In pkgsrc, many variables can be set by the pkgsrc user in the", + "mk.conf file. This file must be included explicitly. If a ?=", + "operator appears before mk.conf has been included, it will not care", + "about the user's preferences, which can result in unexpected", + "behavior.", + "", + "The easiest way to include the mk.conf file is by including the", + "bsd.prefs.mk file, which will take care of everything.") } func (ck MkLineChecker) checkVarassignPlistComment(varname, value string) { @@ -934,7 +960,7 @@ func (ck MkLineChecker) CheckVartype(var } mkline := ck.MkLine - vartype := mkline.VariableType(varname) + vartype := G.Pkgsrc.VariableType(varname) if op == opAssignAppend { if vartype != nil && !vartype.MayBeAppendedTo() { @@ -1008,7 +1034,7 @@ func (ck MkLineChecker) checkText(text s } // Note: A simple -R is not detected, as the rate of false positives is too high. - if m, flag := match1(text, `\b(-Wl,--rpath,|-Wl,-rpath-link,|-Wl,-rpath,|-Wl,-R)\b`); m { + if m, flag := match1(text, `(-Wl,--rpath,|-Wl,-rpath-link,|-Wl,-rpath,|-Wl,-R\b)`); m { mkline.Warnf("Please use ${COMPILER_RPATH_FLAG} instead of %q.", flag) } @@ -1070,7 +1096,7 @@ func (ck MkLineChecker) checkDirectiveCo ck.CheckVartype(varname, opUseMatch, modifier[1:], "") value := modifier[1:] - vartype := mkline.VariableType(varname) + vartype := G.Pkgsrc.VariableType(varname) if matches(value, `^[\w-/]+$`) && vartype != nil && !vartype.IsConsideredList() { mkline.Notef("%s should be compared using == instead of the :M or :N modifier without wildcards.", varname) Explain( @@ -1180,10 +1206,10 @@ func (ck MkLineChecker) CheckRelativePat switch { case !hasPrefix(relativePath, "../"): - case matches(relativePath, `^\.\./\.\./[^/]+/[^/]`): - // From a package to another package. case hasPrefix(relativePath, "../../mk/"): // From a package to the infrastructure. + case matches(relativePath, `^\.\./\.\./[^/]+/[^/]`): + // From a package to another package. case hasPrefix(relativePath, "../mk/") && relpath(path.Dir(mkline.Filename), G.Pkgsrc.File(".")) == "..": // For category Makefiles. default: Index: pkgsrc/pkgtools/pkglint/files/mklines.go diff -u pkgsrc/pkgtools/pkglint/files/mklines.go:1.30 pkgsrc/pkgtools/pkglint/files/mklines.go:1.31 --- pkgsrc/pkgtools/pkglint/files/mklines.go:1.30 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/mklines.go Wed Sep 5 17:56:22 2018 @@ -8,21 +8,18 @@ import ( // MkLines contains data for the Makefile (or *.mk) that is currently checked. type MkLines struct { - mklines []MkLine - lines []Line - forVars map[string]bool // The variables currently used in .for loops - target string // Current make(1) target - vars Scope - buildDefs map[string]bool // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it. - plistVarAdded map[string]MkLine // Identifiers that are added to PLIST_VARS. - plistVarSet map[string]MkLine // Identifiers for which PLIST.${id} is defined. - plistVarSkip bool // True if any of the PLIST_VARS identifiers refers to a variable. - tools map[string]bool // Set of tools that are declared to be used. - toolRegistry ToolRegistry // Tools defined in file scope. - SeenBsdPrefsMk bool - indentation *Indentation // Indentation depth of preprocessing directives; only available during MkLines.ForEach. + mklines []MkLine + lines []Line + forVars map[string]bool // The variables currently used in .for loops + target string // Current make(1) target + vars Scope + buildDefs map[string]bool // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it. + plistVarAdded map[string]MkLine // Identifiers that are added to PLIST_VARS. + plistVarSet map[string]MkLine // Identifiers for which PLIST.${id} is defined. + plistVarSkip bool // True if any of the PLIST_VARS identifiers refers to a variable. + Tools Tools // Tools defined in file scope. + indentation *Indentation // Indentation depth of preprocessing directives; only available during MkLines.ForEach. Once - // XXX: Why both tools and toolRegistry? } func NewMkLines(lines []Line) *MkLines { @@ -30,12 +27,14 @@ func NewMkLines(lines []Line) *MkLines { for i, line := range lines { mklines[i] = NewMkLine(line) } - tools := make(map[string]bool) - G.Pkgsrc.Tools.ForEach(func(tool *Tool) { - if tool.Predefined { - tools[tool.Name] = true - } - }) + + traceName := "MkLines" + if len(lines) != 0 { + traceName = lines[0].Filename + } + + tools := NewTools(traceName) + tools.AddAll(G.Pkgsrc.Tools) return &MkLines{ mklines, @@ -48,8 +47,6 @@ func NewMkLines(lines []Line) *MkLines { make(map[string]MkLine), false, tools, - NewToolRegistry(), - false, nil, Once{}} } @@ -61,13 +58,6 @@ func (mklines *MkLines) UseVar(mkline Mk } } -func (mklines *MkLines) VarValue(varname string) (value string, found bool) { - if mkline := mklines.vars.FirstDefinition(varname); mkline != nil { - return mkline.Value(), true - } - return "", false -} - func (mklines *MkLines) Check() { if trace.Tracing { defer trace.Call1(mklines.lines[0].Filename)() @@ -99,11 +89,17 @@ func (mklines *MkLines) Check() { substcontext := NewSubstContext() var varalign VaralignBlock lastMkline := mklines.mklines[len(mklines.mklines)-1] + isHacksMk := mklines.lines[0].Basename == "hacks.mk" lineAction := func(mkline MkLine) bool { + if isHacksMk { + mklines.Tools.SeenPrefs = true + } + ck := MkLineChecker{mkline} ck.Check() varalign.Check(mkline) + mklines.Tools.ParseToolLine(mkline) switch { case mkline.IsEmpty(): @@ -111,7 +107,7 @@ func (mklines *MkLines) Check() { case mkline.IsVarassign(): mklines.target = "" - mkline.Tokenize(mkline.Value()) // Just for the side-effect of the warning. + mkline.Tokenize(mkline.Value(), true) // Just for the side-effect of the warnings. substcontext.Varassign(mkline) switch mkline.Varcanon() { @@ -132,10 +128,6 @@ func (mklines *MkLines) Check() { case mkline.IsInclude(): mklines.target = "" - switch path.Base(mkline.IncludeFile()) { - case "bsd.prefs.mk", "bsd.fast.prefs.mk", "bsd.builtin.mk": - mklines.setSeenBsdPrefsMk() - } if G.Pkg != nil { G.Pkg.CheckInclude(mkline, mklines.indentation) } @@ -149,7 +141,7 @@ func (mklines *MkLines) Check() { mklines.target = mkline.Targets() case mkline.IsShellCommand(): - mkline.Tokenize(mkline.ShellCommand()) + mkline.Tokenize(mkline.ShellCommand(), true) // Just for the side-effect of the warnings. } return true @@ -162,7 +154,11 @@ func (mklines *MkLines) Check() { } } - mklines.ForEach(lineAction, atEnd) + // TODO: Extract this code so that it is clearly visible in the stack trace. + if trace.Tracing { + trace.Stepf("Starting main checking loop") + } + mklines.ForEachEnd(lineAction, atEnd) substcontext.Finish(lastMkline) varalign.Finish() @@ -174,8 +170,18 @@ func (mklines *MkLines) Check() { // ForEach calls the action for each line, until the action returns false. // It keeps track of the indentation and all conditional variables. -func (mklines *MkLines) ForEach(action func(mkline MkLine) bool, atEnd func(mkline MkLine)) { +func (mklines *MkLines) ForEach(action func(mkline MkLine)) { + mklines.ForEachEnd( + func(mkline MkLine) bool { action(mkline); return true }, + func(mkline MkLine) {}) +} + +// ForEachEnd calls the action for each line, until the action returns false. +// It keeps track of the indentation and all conditional variables. +// At the end, atEnd is called with the last line as its argument. +func (mklines *MkLines) ForEachEnd(action func(mkline MkLine) bool, atEnd func(lastMkline MkLine)) { mklines.indentation = NewIndentation() + mklines.Tools.SeenPrefs = false for _, mkline := range mklines.mklines { mklines.indentation.TrackBefore(mkline) @@ -195,6 +201,8 @@ func (mklines *MkLines) DetermineDefined } for _, mkline := range mklines.mklines { + mklines.Tools.ParseToolLine(mkline) + if !mkline.IsVarassign() { continue } @@ -225,22 +233,6 @@ func (mklines *MkLines) DetermineDefined } } - case "USE_TOOLS": - tools := mkline.Value() - if matches(tools, `\bautoconf213\b`) { - tools += " autoconf autoheader-2.13 autom4te-2.13 autoreconf-2.13 autoscan-2.13 autoupdate-2.13 ifnames-2.13" - } - if matches(tools, `\bautoconf\b`) { - tools += " autoheader autom4te autoreconf autoscan autoupdate ifnames" - } - for _, tool := range splitOnSpace(tools) { - tool = strings.Split(tool, ":")[0] - mklines.tools[tool] = true - if trace.Tracing { - trace.Step1("%s is added to USE_TOOLS.", tool) - } - } - case "SUBST_VARS.*": for _, svar := range splitOnSpace(mkline.Value()) { mklines.UseVar(mkline, varnameCanon(svar)) @@ -255,8 +247,6 @@ func (mklines *MkLines) DetermineDefined defineVar(mkline, osvar) } } - - mklines.toolRegistry.ParseToolLine(mkline) } } @@ -287,9 +277,7 @@ func (mklines *MkLines) collectPlistVars func (mklines *MkLines) collectElse() { // Make a dry-run over the lines, which sets data.elseLine (in mkline.go) as a side-effect. - mklines.ForEach( - func(mkline MkLine) bool { return true }, - func(mkline MkLine) {}) + mklines.ForEach(func(mkline MkLine) {}) } func (mklines *MkLines) DetermineUsedVariables() { @@ -352,15 +340,6 @@ func (mklines *MkLines) determineDocumen finish() } -func (mklines *MkLines) setSeenBsdPrefsMk() { - if !mklines.SeenBsdPrefsMk { - mklines.SeenBsdPrefsMk = true - if trace.Tracing { - trace.Stepf("Mk.setSeenBsdPrefsMk") - } - } -} - func (mklines *MkLines) CheckRedundantVariables() { scope := NewRedundantScope() isRelevant := func(old, new MkLine) bool { @@ -388,12 +367,7 @@ func (mklines *MkLines) CheckRedundantVa } } - mklines.ForEach( - func(mkline MkLine) bool { - scope.Handle(mkline) - return true - }, - func(mkline MkLine) {}) + mklines.ForEach(scope.Handle) } func (mklines *MkLines) SaveAutofixChanges() { Index: pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go diff -u pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go:1.30 pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go:1.31 --- pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go:1.30 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go Wed Sep 5 17:56:22 2018 @@ -7,61 +7,70 @@ import ( ) func (s *Suite) Test_VartypeCheck_AwkCommand(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).AwkCommand) - runVartypeChecks(t, "PLIST_AWK", opAssignAppend, (*VartypeCheck).AwkCommand, + vt.Varname("PLIST_AWK") + vt.Op(opAssignAppend) + vt.Values( "{print $0}", "{print $$0}") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: $0 is ambiguous. Use ${0} if you mean a Makefile variable or $$0 if you mean a shell variable.") } func (s *Suite) Test_VartypeCheck_BasicRegularExpression(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).BasicRegularExpression) - runVartypeChecks(t, "REPLACE_FILES.pl", opAssign, (*VartypeCheck).BasicRegularExpression, + vt.Varname("REPLACE_FILES.pl") + vt.Values( ".*\\.pl$", ".*\\.pl$$") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: Pkglint parse error in MkLine.Tokenize at \"$\".") } func (s *Suite) Test_VartypeCheck_BuildlinkDepmethod(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).BuildlinkDepmethod) - runVartypeChecks(t, "BUILDLINK_DEPMETHOD.libc", opAssignDefault, (*VartypeCheck).BuildlinkDepmethod, + vt.Varname("BUILDLINK_DEPMETHOD.libc") + vt.Op(opAssignDefault) + vt.Values( "full", "unknown") - t.CheckOutputLines( + vt.Output( "WARN: fname:2: Invalid dependency method \"unknown\". Valid methods are \"build\" or \"full\".") } func (s *Suite) Test_VartypeCheck_Category(c *check.C) { t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).Category) t.SetupFileLines("filesyscategory/Makefile", "# empty") t.SetupFileLines("wip/Makefile", "# empty") - runVartypeChecks(t, "CATEGORIES", opAssign, (*VartypeCheck).Category, + vt.Varname("CATEGORIES") + vt.Values( "chinese", "arabic", "filesyscategory", "wip") - t.CheckOutputLines( + vt.Output( "ERROR: fname:2: Invalid category \"arabic\".", "ERROR: fname:4: Invalid category \"wip\".") } func (s *Suite) Test_VartypeCheck_CFlag(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).CFlag) - runVartypeChecks(t, "CFLAGS", opAssignAppend, (*VartypeCheck).CFlag, + vt.Varname("CFLAGS") + vt.Op(opAssignAppend) + vt.Values( "-Wall", "/W3", "target:sparc64", @@ -69,19 +78,27 @@ func (s *Suite) Test_VartypeCheck_CFlag( "-XX:+PrintClassHistogramAfterFullGC", "`pkg-config pidgin --cflags`") - t.CheckOutputLines( + vt.Output( "WARN: fname:2: Compiler flag \"/W3\" should start with a hyphen.", "WARN: fname:3: Compiler flag \"target:sparc64\" should start with a hyphen.", "WARN: fname:5: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".") + + vt.Op(opUseMatch) + vt.Values( + "anything") + + vt.OutputEmpty() } func (s *Suite) Test_VartypeCheck_Comment(c *check.C) { t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).Comment) G.Pkg = NewPackage(t.File("category/converter")) G.Pkg.EffectivePkgbase = "converter" - runVartypeChecks(t, "COMMENT", opAssign, (*VartypeCheck).Comment, + vt.Varname("COMMENT") + vt.Values( "Versatile Programming Language", "TODO: Short description of the package", "A great package.", @@ -93,7 +110,7 @@ func (s *Suite) Test_VartypeCheck_Commen "The Big New Package is a great package", "Converter converts between measurement units") - t.CheckOutputLines( + vt.Output( "ERROR: fname:2: COMMENT must be set.", "WARN: fname:3: COMMENT should not begin with \"A\".", "WARN: fname:3: COMMENT should not end with a period.", @@ -108,16 +125,18 @@ func (s *Suite) Test_VartypeCheck_Commen } func (s *Suite) Test_VartypeCheck_ConfFiles(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).ConfFiles) - runVartypeChecks(t, "CONF_FILES", opAssignAppend, (*VartypeCheck).ConfFiles, + vt.Varname("CONF_FILES") + vt.Op(opAssignAppend) + vt.Values( "single/file", "share/etc/config ${PKG_SYSCONFDIR}/etc/config", "share/etc/config ${PKG_SYSCONFBASE}/etc/config file", "share/etc/config ${PREFIX}/etc/config share/etc/config2 ${VARBASE}/config2", "share/etc/bootrc /etc/bootrc") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: Values for CONF_FILES should always be pairs of paths.", "WARN: fname:3: Values for CONF_FILES should always be pairs of paths.", "WARN: fname:5: Found absolute pathname: /etc/bootrc", @@ -125,9 +144,11 @@ func (s *Suite) Test_VartypeCheck_ConfFi } func (s *Suite) Test_VartypeCheck_Dependency(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Dependency) - runVartypeChecks(t, "CONFLICTS", opAssignAppend, (*VartypeCheck).Dependency, + vt.Varname("CONFLICTS") + vt.Op(opAssignAppend) + vt.Values( "Perl", "perl5>=5.22", "perl5-*", @@ -150,7 +171,7 @@ func (s *Suite) Test_VartypeCheck_Depend "{ssh{,6}-[0-9]*,openssh-[0-9]*}", "gnome-control-center>=2.20.1{,nb*}") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: Unknown dependency pattern \"Perl\".", "WARN: fname:3: Please use \"perl5-[0-9]*\" instead of \"perl5-*\".", "WARN: fname:5: Only [0-9]* is allowed in the numeric part of a dependency.", @@ -165,24 +186,18 @@ func (s *Suite) Test_VartypeCheck_Depend func (s *Suite) Test_VartypeCheck_DependencyWithPath(c *check.C) { t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).DependencyWithPath) t.CreateFileLines("x11/alacarte/Makefile") t.CreateFileLines("category/package/Makefile") + t.CreateFileLines("devel/gettext/Makefile") + t.CreateFileLines("devel/gmake/Makefile") G.Pkg = NewPackage(t.File("category/package")) - // Since this test involves relative paths, the filename of the line must be realistic. - // Therefore this custom implementation of runVartypeChecks. - runChecks := func(values ...string) { - for i, value := range values { - mkline := t.NewMkLine(G.Pkg.File("fname.mk"), i+1, "DEPENDS+=\t"+value) - mkline.Tokenize(mkline.Value()) - valueNovar := mkline.WithoutMakeVariables(mkline.Value()) - vc := &VartypeCheck{mkline, mkline.Line, mkline.Varname(), mkline.Op(), mkline.Value(), valueNovar, "", false} - (*VartypeCheck).DependencyWithPath(vc) - } - } - - runChecks( + vt.Varname("DEPENDS") + vt.Op(opAssignAppend) + vt.File(G.Pkg.File("fname.mk")) + vt.Values( "Perl", "perl5>=5.22:../perl5", "perl5>=5.24:../../lang/perl5", @@ -194,9 +209,11 @@ func (s *Suite) Test_VartypeCheck_Depend "broken=:../../x11/alacarte", "broken-:../../x11/alacarte", "broken>:../../x11/alacarte", - "gtk2+>=2.16:../../x11/alacarte") + "gtk2+>=2.16:../../x11/alacarte", + "gettext-[0-9]*:../../devel/gettext", + "gmake-[0-9]*:../../devel/gmake") - t.CheckOutputLines( + vt.Output( "WARN: ~/category/package/fname.mk:1: Unknown dependency pattern with path \"Perl\".", "WARN: ~/category/package/fname.mk:2: Dependencies should have the form \"../../category/package\".", "ERROR: ~/category/package/fname.mk:3: \"../../lang/perl5\" does not exist.", @@ -209,29 +226,33 @@ func (s *Suite) Test_VartypeCheck_Depend "WARN: ~/category/package/fname.mk:8: Unknown dependency pattern \"broken=0\".", "WARN: ~/category/package/fname.mk:9: Unknown dependency pattern \"broken=\".", "WARN: ~/category/package/fname.mk:10: Unknown dependency pattern \"broken-\".", - "WARN: ~/category/package/fname.mk:11: Unknown dependency pattern \"broken>\".") + "WARN: ~/category/package/fname.mk:11: Unknown dependency pattern \"broken>\".", + "WARN: ~/category/package/fname.mk:13: Please use USE_TOOLS+=msgfmt instead of this dependency.", + "WARN: ~/category/package/fname.mk:14: Please use USE_TOOLS+=gmake instead of this dependency.") } func (s *Suite) Test_VartypeCheck_DistSuffix(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).DistSuffix) - runVartypeChecks(t, "EXTRACT_SUFX", opAssign, (*VartypeCheck).DistSuffix, + vt.Varname("EXTRACT_SUFX") + vt.Values( ".tar.gz", ".tar.bz2") - t.CheckOutputLines( + vt.Output( "NOTE: fname:1: EXTRACT_SUFX is \".tar.gz\" by default, so this definition may be redundant.") } func (s *Suite) Test_VartypeCheck_EmulPlatform(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).EmulPlatform) - runVartypeChecks(t, "EMUL_PLATFORM", opAssign, (*VartypeCheck).EmulPlatform, + vt.Varname("EMUL_PLATFORM") + vt.Values( "linux-i386", "nextbsd-8087", "${LINUX}") - t.CheckOutputLines( + vt.Output( "WARN: fname:2: \"nextbsd\" is not valid for the operating system part of EMUL_PLATFORM. "+ "Use one of "+ "{ bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux "+ @@ -249,16 +270,21 @@ func (s *Suite) Test_VartypeCheck_EmulPl } func (s *Suite) Test_VartypeCheck_Enum(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), enum("jdk1 jdk2 jdk4").checker) - runVartypeMatchChecks(t, "JDK", enum("jdk1 jdk2 jdk4").checker, + vt.Varname("JDK") + vt.Op(opUseMatch) + vt.Values( "*", "jdk*", "sun-jdk*", - "${JDKNAME}") + "${JDKNAME}", + "[") - t.CheckOutputLines( - "WARN: fname:3: The pattern \"sun-jdk*\" cannot match any of { jdk1 jdk2 jdk4 } for JDK.") + vt.Output( + "WARN: fname:3: The pattern \"sun-jdk*\" cannot match any of { jdk1 jdk2 jdk4 } for JDK.", + "WARN: fname:5: Invalid match pattern \"[\".", + "WARN: fname:5: The pattern \"[\" cannot match any of { jdk1 jdk2 jdk4 } for JDK.") } func (s *Suite) Test_VartypeCheck_Enum__use_match(c *check.C) { @@ -285,17 +311,19 @@ func (s *Suite) Test_VartypeCheck_Enum__ func (s *Suite) Test_VartypeCheck_FetchURL(c *check.C) { t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).FetchURL) t.SetupMasterSite("MASTER_SITE_GNU", "http://ftp.gnu.org/pub/gnu/") t.SetupMasterSite("MASTER_SITE_GITHUB", "https://github.com/") - runVartypeChecks(t, "MASTER_SITES", opAssign, (*VartypeCheck).FetchURL, + vt.Varname("MASTER_SITES") + vt.Values( "https://github.com/example/project/", "http://ftp.gnu.org/pub/gnu/bison", // Missing a slash at the end "${MASTER_SITE_GNU:=bison}", "${MASTER_SITE_INVALID:=subdir/}") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: Please use ${MASTER_SITE_GITHUB:=example/} "+ "instead of \"https://github.com/example/project/\" "+ "and run \""+confMake+" help topic=github\" for further tips.", @@ -304,73 +332,201 @@ func (s *Suite) Test_VartypeCheck_FetchU "ERROR: fname:4: The site MASTER_SITE_INVALID does not exist.") // PR 46570, keyword gimp-fix-ca - runVartypeChecks(t, "MASTER_SITES", opAssign, (*VartypeCheck).FetchURL, + vt.Values( "https://example.org/download.cgi?fname=fname&sha1=12341234") t.CheckOutputEmpty() - runVartypeChecks(t, "MASTER_SITES", opAssign, (*VartypeCheck).FetchURL, + vt.Values( "http://example.org/distfiles/", "http://example.org/download?fname=distfile;version=1.0", "http://example.org/download?fname=;version=") - t.CheckOutputLines( - "WARN: fname:3: \"http://example.org/download?fname=;version=\" is not a valid URL.") + vt.Output( + "WARN: fname:8: \"http://example.org/download?fname=;version=\" is not a valid URL.") } func (s *Suite) Test_VartypeCheck_Filename(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Filename) - runVartypeChecks(t, "FNAME", opAssign, (*VartypeCheck).Filename, + vt.Varname("FNAME") + vt.Values( "Filename with spaces.docx", "OS/2-manual.txt") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: \"Filename with spaces.docx\" is not a valid filename.", "WARN: fname:2: A filename should not contain a slash.") + + vt.Op(opUseMatch) + vt.Values( + "Filename with spaces.docx") + + // There's no guarantee that a file name only contains [A-Za-z0-9.]. + // Therefore there are no useful checks in this situation. + vt.OutputEmpty() } -func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { +func (s *Suite) Test_VartypeCheck_Filemask(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Filemask) + + vt.Varname("FNAME") + vt.Values( + "Filemask with spaces.docx", + "OS/2-manual.txt") + + vt.Output( + "WARN: fname:1: \"Filemask with spaces.docx\" is not a valid filename mask.", + "WARN: fname:2: A filename mask should not contain a slash.") + + vt.Op(opUseMatch) + vt.Values( + "Filemask with spaces.docx") + + // There's no guarantee that a file name only contains [A-Za-z0-9.]. + // Therefore there are no useful checks in this situation. + vt.OutputEmpty() +} + +func (s *Suite) Test_VartypeCheck_FileMode(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).FileMode) + + vt.Varname("HIGHSCORE_PERMS") + vt.Values( + "u+rwx", + "0600", + "1234", + "12345", + "${OTHER_PERMS}") + + vt.Output( + "WARN: fname:1: Invalid file mode \"u+rwx\".", + "WARN: fname:4: Invalid file mode \"12345\".") + + vt.Op(opUseMatch) + vt.Values( + "u+rwx") + + // There's no guarantee that a file name only contains [A-Za-z0-9.]. + // Therefore there are no useful checks in this situation. + vt.Output( + "WARN: fname:11: Invalid file mode \"u+rwx\".") +} + +func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { + t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).Homepage) + + vt.Varname("HOMEPAGE") + vt.Values( + "${MASTER_SITES}") + + vt.Output( + "WARN: fname:1: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + + G.Pkg = NewPackage(t.File("category/package")) + G.Pkg.vars.Define("MASTER_SITES", t.NewMkLine(G.Pkg.File("Makefile"), 5, "MASTER_SITES=\thttps://cdn.NetBSD.org/pub/pkgsrc/distfiles/")) + + vt.Values( + "${MASTER_SITES}") + + vt.Output( + "WARN: fname:2: HOMEPAGE should not be defined in terms of MASTER_SITEs. Use https://cdn.NetBSD.org/pub/pkgsrc/distfiles/ directly.") +} + +func (s *Suite) Test_VartypeCheck_Identifier(c *check.C) { + t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).Identifier) + + vt.Varname("SUBST_CLASSES") + vt.Values( + "${OTHER_VAR}", + "identifiers cannot contain spaces", + "id/cannot/contain/slashes") + vt.Op(opUseMatch) + vt.Values( + "[A-Z]", + "A*B") + + vt.Output( + "WARN: fname:2: Invalid identifier \"identifiers cannot contain spaces\".", + "WARN: fname:3: Invalid identifier \"id/cannot/contain/slashes\".", + "WARN: fname:11: Invalid identifier pattern \"[A-Z]\" for SUBST_CLASSES.") +} + +func (s *Suite) Test_VartypeCheck_Integer(c *check.C) { t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).Integer) + + vt.Varname("NUMBER") + vt.Values( + "${OTHER_VAR}", + "123", + "-13", + "11111111111111111111111111111111111111111111111") + + vt.Output( + "WARN: fname:1: Invalid integer \"${OTHER_VAR}\".", + "WARN: fname:3: Invalid integer \"-13\".") +} + +func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).LdFlag) - runVartypeChecks(t, "LDFLAGS", opAssignAppend, (*VartypeCheck).LdFlag, + vt.Varname("LDFLAGS") + vt.Op(opAssignAppend) + vt.Values( "-lc", "-L/usr/lib64", "`pkg-config pidgin --ldflags`", - "-unknown") - - t.CheckOutputLines( - "WARN: fname:4: Unknown linker flag \"-unknown\".") + "-unknown", + "no-hyphen", + "-Wl,--rpath,/usr/lib64") + vt.Op(opUseMatch) + vt.Values( + "anything") + + vt.Output( + "WARN: fname:4: Unknown linker flag \"-unknown\".", + "WARN: fname:5: Linker flag \"no-hyphen\" should start with a hyphen.", + "WARN: fname:6: Please use \"${COMPILER_RPATH_FLAG}\" instead of \"-Wl,--rpath\".") } func (s *Suite) Test_VartypeCheck_License(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).License) - runVartypeChecks(t, "LICENSE", opAssign, (*VartypeCheck).License, + vt.Varname("LICENSE") + vt.Values( "gnu-gpl-v2", "AND mit") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: License file ~/licenses/gnu-gpl-v2 does not exist.", "ERROR: fname:2: Parse error for license condition \"AND mit\".") - runVartypeChecks(t, "LICENSE", opAssignAppend, (*VartypeCheck).License, + vt.Op(opAssignAppend) + vt.Values( "gnu-gpl-v2", "AND mit") - t.CheckOutputLines( - "ERROR: fname:1: Parse error for appended license condition \"gnu-gpl-v2\".", - "WARN: fname:2: License file ~/licenses/mit does not exist.") + vt.Output( + "ERROR: fname:11: Parse error for appended license condition \"gnu-gpl-v2\".", + "WARN: fname:12: License file ~/licenses/mit does not exist.") } func (s *Suite) Test_VartypeCheck_MachineGnuPlatform(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).MachineGnuPlatform) - runVartypeMatchChecks(t, "MACHINE_GNU_PLATFORM", (*VartypeCheck).MachineGnuPlatform, + vt.Varname("MACHINE_GNU_PLATFORM") + vt.Op(opUseMatch) + vt.Values( "x86_64-pc-cygwin", - "Cygwin-*-amd64") + "Cygwin-*-amd64", + "x86_64-*", + "*-*-*-*", + "${OTHER_VAR}") - t.CheckOutputLines( + vt.Output( "WARN: fname:2: The pattern \"Cygwin\" cannot match any of "+ "{ aarch64 aarch64_be alpha amd64 arc arm armeb armv4 armv4eb armv6 armv6eb armv7 armv7eb "+ "cobalt convex dreamcast hpcmips hpcsh hppa hppa64 i386 i486 ia64 m5407 m68010 m68k m88k "+ @@ -380,73 +536,143 @@ func (s *Suite) Test_VartypeCheck_Machin "WARN: fname:2: The pattern \"amd64\" cannot match any of "+ "{ bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux interix irix linux mirbsd "+ "netbsd openbsd osf1 solaris sunos } "+ - "for the operating system part of MACHINE_GNU_PLATFORM.") + "for the operating system part of MACHINE_GNU_PLATFORM.", + "WARN: fname:4: \"*-*-*-*\" is not a valid platform pattern.") } func (s *Suite) Test_VartypeCheck_MailAddress(c *check.C) { - t := s.Init(c) - - runVartypeChecks(t, "MAINTAINER", opAssign, (*VartypeCheck).MailAddress, - "pkgsrc-users@netbsd.org") + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).MailAddress) - t.CheckOutputLines( - "WARN: fname:1: Please write \"NetBSD.org\" instead of \"netbsd.org\".") + vt.Varname("MAINTAINER") + vt.Values( + "pkgsrc-users@netbsd.org", + "tech-pkg@NetBSD.org", + "packages@NetBSD.org", + "user1@example.org,user2@example.org") + + vt.Output( + "WARN: fname:1: Please write \"NetBSD.org\" instead of \"netbsd.org\".", + "ERROR: fname:2: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", + "ERROR: fname:3: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", + "WARN: fname:4: \"user1@example.org,user2@example.org\" is not a valid mail address.") } func (s *Suite) Test_VartypeCheck_Message(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Message) - runVartypeChecks(t, "SUBST_MESSAGE.id", opAssign, (*VartypeCheck).Message, + vt.Varname("SUBST_MESSAGE.id") + vt.Values( "\"Correct paths\"", "Correct paths") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: SUBST_MESSAGE.id should not be quoted.") } func (s *Suite) Test_VartypeCheck_Option(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Option) G.Pkgsrc.PkgOptions["documented"] = "Option description" G.Pkgsrc.PkgOptions["undocumented"] = "" - runVartypeChecks(t, "PKG_OPTIONS.pkgbase", opAssign, (*VartypeCheck).Option, + vt.Varname("PKG_OPTIONS.pkgbase") + vt.Values( "documented", "undocumented", - "unknown") - - t.CheckOutputLines( - "WARN: fname:3: Unknown option \"unknown\".") + "unknown", + "underscore_is_deprecated", + "UPPER") + + vt.Output( + "WARN: fname:3: Unknown option \"unknown\".", + "WARN: fname:4: Use of the underscore character in option names is deprecated.", + "ERROR: fname:5: Invalid option name \"UPPER\". "+ + "Option names must start with a lowercase letter and be all-lowercase.") } func (s *Suite) Test_VartypeCheck_Pathlist(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Pathlist) - runVartypeChecks(t, "PATH", opAssign, (*VartypeCheck).Pathlist, - "/usr/bin:/usr/sbin:.::${LOCALBASE}/bin:${HOMEPAGE:S,https://,,}") + vt.Varname("PATH") + vt.Values( + "/usr/bin:/usr/sbin:.::${LOCALBASE}/bin:${HOMEPAGE:S,https://,,}", + "/directory with spaces") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: All components of PATH (in this case \".\") should be absolute paths.", - "WARN: fname:1: All components of PATH (in this case \"\") should be absolute paths.") + "WARN: fname:1: All components of PATH (in this case \"\") should be absolute paths.", + "WARN: fname:2: \"/directory with spaces\" is not a valid pathname.") +} + +func (s *Suite) Test_VartypeCheck_Pathmask(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Pathmask) + + vt.Varname("DISTDIRS") + vt.Values( + "/home/user/*", + "src/*&*", + "src/*/*") + + vt.Output( + "WARN: fname:1: Found absolute pathname: /home/user/*", + "WARN: fname:2: \"src/*&*\" is not a valid pathname mask.") + + vt.Op(opUseMatch) + vt.Values("any") + + vt.OutputEmpty() +} + +func (s *Suite) Test_VartypeCheck_Pathname(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Pathname) + + vt.Varname("EGDIR") + vt.Values( + "${PREFIX}/*", + "${PREFIX}/share/locale", + "share/locale", + "/bin") + vt.Op(opUseMatch) + vt.Values( + "anything") + + vt.Output( + "WARN: fname:1: \"${PREFIX}/*\" is not a valid pathname.", + "WARN: fname:4: Found absolute pathname: /bin") +} + +func (s *Suite) Test_VartypeCheck_Perl5Packlist(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Perl5Packlist) + + vt.Varname("PERL5_PACKLIST") + vt.Values( + "${PKGBASE}", + "anything else") + + vt.Output( + "WARN: fname:1: PERL5_PACKLIST should not depend on other variables.") } func (s *Suite) Test_VartypeCheck_Perms(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Perms) - runVartypeChecks(t, "CONF_FILES_PERMS", opAssignAppend, (*VartypeCheck).Perms, + vt.Varname("CONF_FILES_PERMS") + vt.Op(opAssignAppend) + vt.Values( "root", "${ROOT_USER}", "ROOT_USER", "${REAL_ROOT_USER}") - t.CheckOutputLines( + vt.Output( "ERROR: fname:2: ROOT_USER must not be used in permission definitions. Use REAL_ROOT_USER instead.") } func (s *Suite) Test_VartypeCheck_Pkgname(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PkgName) - runVartypeChecks(t, "PKGNAME", opAssign, (*VartypeCheck).PkgName, + vt.Varname("PKGNAME") + vt.Values( "pkgbase-0", "pkgbase-1.0", "pkgbase-1.1234567890", @@ -457,43 +683,71 @@ func (s *Suite) Test_VartypeCheck_Pkgnam "pkgbase-z1", "pkgbase-3.1.4.1.5.9.2.6.5.3.5.8.9.7.9") - t.CheckOutputLines( + vt.Output( "WARN: fname:8: \"pkgbase-z1\" is not a valid package name. " + "A valid package name has the form packagename-version, " + "where version consists only of digits, letters and dots.") } func (s *Suite) Test_VartypeCheck_PkgOptionsVar(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PkgOptionsVar) - runVartypeChecks(t, "PKG_OPTIONS_VAR.screen", opAssign, (*VartypeCheck).PkgOptionsVar, + vt.Varname("PKG_OPTIONS_VAR.screen") + vt.Values( "PKG_OPTIONS.${PKGBASE}", - "PKG_OPTIONS.anypkgbase") + "PKG_OPTIONS.anypkgbase", + "PKG_OPTS.mc") - t.CheckOutputLines( - "ERROR: fname:1: PKGBASE must not be used in PKG_OPTIONS_VAR.") + vt.Output( + "ERROR: fname:1: PKGBASE must not be used in PKG_OPTIONS_VAR.", + "ERROR: fname:3: PKG_OPTIONS_VAR must be of the form \"PKG_OPTIONS.*\", not \"PKG_OPTS.mc\".") } -func (s *Suite) Test_VartypeCheck_PkgRevision(c *check.C) { +func (s *Suite) Test_VartypeCheck_PkgPath(c *check.C) { t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).PkgPath) + + t.CreateFileLines("category/other-package/Makefile") + t.Chdir("category/package") + + vt.Varname("PKGPATH") + vt.Values( + "category/other-package", + "${OTHER_VAR}", + "invalid", + "../../invalid/relative") + + vt.Output( + "ERROR: fname:3: \"../../invalid\" does not exist.", + "WARN: fname:3: \"../../invalid\" is not a valid relative package directory.", + "ERROR: fname:4: \"../../../../invalid/relative\" does not exist.", + "WARN: fname:4: \"../../../../invalid/relative\" is not a valid relative package directory.") +} - runVartypeChecks(t, "PKGREVISION", opAssign, (*VartypeCheck).PkgRevision, +func (s *Suite) Test_VartypeCheck_PkgRevision(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PkgRevision) + + vt.Varname("PKGREVISION") + vt.Values( "3a") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: PKGREVISION must be a positive integer number.", "ERROR: fname:1: PKGREVISION only makes sense directly in the package Makefile.") - runVartypeChecksFname(t, "Makefile", "PKGREVISION", opAssign, (*VartypeCheck).PkgRevision, + vt.File("Makefile") + vt.Values( "3") - t.CheckOutputEmpty() + vt.OutputEmpty() } func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).MachinePlatformPattern) - runVartypeMatchChecks(t, "ONLY_FOR_PLATFORM", (*VartypeCheck).MachinePlatformPattern, + vt.Varname("ONLY_FOR_PLATFORM") + vt.Op(opUseMatch) + vt.Values( "linux-i386", "nextbsd-5.0-8087", "netbsd-7.0-l*", @@ -502,7 +756,7 @@ func (s *Suite) Test_VartypeCheck_Machin "FreeBSD-*", "${LINUX}") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: \"linux-i386\" is not a valid platform pattern.", "WARN: fname:2: The pattern \"nextbsd\" cannot match any of "+ "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ @@ -530,199 +784,449 @@ func (s *Suite) Test_VartypeCheck_Machin } func (s *Suite) Test_VartypeCheck_PythonDependency(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PythonDependency) - runVartypeChecks(t, "PYTHON_VERSIONED_DEPENDENCIES", opAssign, (*VartypeCheck).PythonDependency, + vt.Varname("PYTHON_VERSIONED_DEPENDENCIES") + vt.Values( "cairo", "${PYDEP}", "cairo,X") - t.CheckOutputLines( + vt.Output( "WARN: fname:2: Python dependencies should not contain variables.", "WARN: fname:3: Invalid Python dependency \"cairo,X\".") } -func (s *Suite) Test_VartypeCheck_Restricted(c *check.C) { +func (s *Suite) Test_VartypeCheck_PrefixPathname(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PrefixPathname) + + vt.Varname("PKGMANDIR") + vt.Values( + "man/man1", + "share/locale") + + vt.Output( + "WARN: fname:1: Please use \"${PKGMANDIR}/man1\" instead of \"man/man1\".") +} + +func (s *Suite) Test_VartypeCheck_RelativePkgPath(c *check.C) { t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).RelativePkgPath) + + t.CreateFileLines("category/other-package/Makefile") + t.Chdir("category/package") + + vt.Varname("DISTINFO_FILE") + vt.Values( + "category/other-package", + "../../category/other-package", + "${OTHER_VAR}", + "invalid", + "../../invalid/relative") + + vt.Output( + "ERROR: fname:1: \"category/other-package\" does not exist.", + "ERROR: fname:4: \"invalid\" does not exist.", + "ERROR: fname:5: \"../../invalid/relative\" does not exist.") +} + +func (s *Suite) Test_VartypeCheck_Restricted(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Restricted) - runVartypeChecks(t, "NO_BIN_ON_CDROM", opAssign, (*VartypeCheck).Restricted, + vt.Varname("NO_BIN_ON_CDROM") + vt.Values( "May only be distributed free of charge") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: The only valid value for NO_BIN_ON_CDROM is ${RESTRICTED}.") } func (s *Suite) Test_VartypeCheck_SedCommands(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).SedCommands) - runVartypeChecks(t, "SUBST_SED.dummy", opAssign, (*VartypeCheck).SedCommands, + vt.Varname("SUBST_SED.dummy") + vt.Values( "s,@COMPILER@,gcc,g", "-e s,a,b, -e a,b,c,", "-e \"s,#,comment ,\"", - "-e \"s,\\#,comment ,\"") + "-e \"s,\\#,comment ,\"", + "-E", + "-n", + "-e 1d", + "1d", + "-e") - t.CheckOutputLines( + vt.Output( "NOTE: fname:1: Please always use \"-e\" in sed commands, even if there is only one substitution.", "NOTE: fname:2: Each sed command should appear in an assignment of its own.", - "WARN: fname:3: The # character starts a comment.") + "WARN: fname:3: The # character starts a comment.", + "ERROR: fname:3: Invalid shell words \"\\\"s,\" in sed commands.", + "WARN: fname:8: Unknown sed command \"1d\".", + "ERROR: fname:9: The -e option to sed requires an argument.") +} + +func (s *Suite) Test_VartypeCheck_ShellCommand(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).ShellCommand) + + vt.Varname("INSTALL_CMD") + vt.Values( + "${INSTALL_DATA} -m 0644 ${WRKDIR}/source ${DESTDIR}${PREFIX}/target") + + vt.OutputEmpty() } func (s *Suite) Test_VartypeCheck_ShellCommands(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).ShellCommands) - runVartypeChecks(t, "GENERATE_PLIST", opAssign, (*VartypeCheck).ShellCommands, + vt.Varname("GENERATE_PLIST") + vt.Values( "echo bin/program", "echo bin/program;") - t.CheckOutputLines( + vt.Output( "WARN: fname:1: This shell command list should end with a semicolon.") } func (s *Suite) Test_VartypeCheck_Stage(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Stage) - runVartypeChecks(t, "SUBST_STAGE.dummy", opAssign, (*VartypeCheck).Stage, + vt.Varname("SUBST_STAGE.dummy") + vt.Values( "post-patch", "post-modern", "pre-test") - t.CheckOutputLines( + vt.Output( "WARN: fname:2: Invalid stage name \"post-modern\". " + "Use one of {pre,do,post}-{extract,patch,configure,build,test,install}.") } func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).Tool) - t.SetupTool(&Tool{Name: "tool1", Predefined: true}) - t.SetupTool(&Tool{Name: "tool2", Predefined: true}) - t.SetupTool(&Tool{Name: "tool3", Predefined: true}) - - runVartypeChecks(t, "USE_TOOLS", opAssignAppend, (*VartypeCheck).Tool, + t.SetupToolUsable("tool1", "") + t.SetupToolUsable("tool2", "") + t.SetupToolUsable("tool3", "") + + vt.Varname("USE_TOOLS") + vt.Op(opAssignAppend) + vt.Values( "tool3:run", "tool2:unknown", "${t}", - "mal:formed:tool") + "mal:formed:tool", + "unknown") - t.CheckOutputLines( + vt.Output( "ERROR: fname:2: Unknown tool dependency \"unknown\". "+ "Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".", - "ERROR: fname:4: Malformed tool dependency: \"mal:formed:tool\".") + "ERROR: fname:4: Malformed tool dependency: \"mal:formed:tool\".", + "ERROR: fname:5: Unknown tool \"unknown\".") - runVartypeChecks(t, "USE_TOOLS.NetBSD", opAssignAppend, (*VartypeCheck).Tool, + vt.Varname("USE_TOOLS.NetBSD") + vt.Op(opAssignAppend) + vt.Values( "tool3:run", "tool2:unknown") - t.CheckOutputLines( - "ERROR: fname:2: Unknown tool dependency \"unknown\". " + + vt.Output( + "ERROR: fname:12: Unknown tool dependency \"unknown\". " + "Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".") + + vt.Varname("TOOLS_NOOP") + vt.Op(opAssignAppend) + vt.Values( + "gmake:run") + + vt.OutputEmpty() +} + +func (s *Suite) Test_VartypeCheck_URL(c *check.C) { + t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).URL) + + vt.Varname("HOMEPAGE") + vt.Values( + "# none", + "${OTHER_VAR}", + "https://www.netbsd.org/", + "mailto:someone@example.org", + "httpxs://www.example.org", + "https://www.example.org", + "https://www.example.org/path with spaces", + "string with spaces") + + vt.Output( + "WARN: fname:3: Please write NetBSD.org instead of www.netbsd.org.", + "WARN: fname:4: \"mailto:someone@example.org\" is not a valid URL.", + "WARN: fname:5: \"httpxs://www.example.org\" is not a valid URL. Only ftp, gopher, http, and https URLs are allowed here.", + "NOTE: fname:6: For consistency, please add a trailing slash to \"https://www.example.org\".", + "WARN: fname:7: \"https://www.example.org/path with spaces\" is not a valid URL.", + "WARN: fname:8: \"string with spaces\" is not a valid URL.") +} + +func (s *Suite) Test_VartypeCheck_UserGroupName(c *check.C) { + t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).UserGroupName) + + vt.Varname("APACHE_USER") + vt.Values( + "user with spaces", + "typical_username", + "user123", + "domain\\user", + "${OTHER_VAR}") + + vt.Output( + "WARN: fname:1: Invalid user or group name \"user with spaces\".", + "WARN: fname:4: Invalid user or group name \"domain\\\\user\".") } func (s *Suite) Test_VartypeCheck_VariableName(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).VariableName) - runVartypeChecks(t, "BUILD_DEFS", opAssign, (*VartypeCheck).VariableName, + vt.Varname("BUILD_DEFS") + vt.Values( "VARBASE", "VarBase", "PKG_OPTIONS_VAR.pkgbase", "${INDIRECT}") - t.CheckOutputLines( + vt.Output( "WARN: fname:2: \"VarBase\" is not a valid variable name.") } func (s *Suite) Test_VartypeCheck_Version(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Version) - runVartypeChecks(t, "PERL5_REQD", opAssignAppend, (*VartypeCheck).Version, + vt.Varname("PERL5_REQD") + vt.Op(opAssignAppend) + vt.Values( "0", "1.2.3.4.5.6", "4.1nb17", "4.1-SNAPSHOT", "4pre7") - - t.CheckOutputLines( + vt.Output( "WARN: fname:4: Invalid version number \"4.1-SNAPSHOT\".") + + vt.Op(opUseMatch) + vt.Values( + "a*", + "1.2/456", + "[0-9]*") + vt.Output( + "WARN: fname:11: Invalid version number pattern \"a*\".", + "WARN: fname:12: Invalid version number pattern \"1.2/456\".") +} + +func (s *Suite) Test_VartypeCheck_WrapperReorder(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).WrapperReorder) + + vt.Varname("WRAPPER_REORDER") + vt.Op(opAssignAppend) + vt.Values( + "reorder:l:first:second", + "reorder:l:first", + "omit:first") + vt.Output( + "WARN: fname:2: Unknown wrapper reorder command \"reorder:l:first\".", + "WARN: fname:3: Unknown wrapper reorder command \"omit:first\".") +} + +func (s *Suite) Test_VartypeCheck_WrapperTransform(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).WrapperTransform) + + vt.Varname("WRAPPER_TRANSFORM") + vt.Op(opAssignAppend) + vt.Values( + "rm:-O3", + "opt:-option", + "rename:src:dst", + "rm-optarg:-option", + "rmdir:/usr/include", + "rpath:/usr/lib:/usr/pkg/lib", + "rpath:/usr/lib", + "unknown") + vt.Output( + "WARN: fname:7: Unknown wrapper transform command \"rpath:/usr/lib\".", + "WARN: fname:8: Unknown wrapper transform command \"unknown\".") +} + +func (s *Suite) Test_VartypeCheck_WrksrcSubdirectory(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).WrksrcSubdirectory) + + vt.Varname("BUILD_DIRS") + vt.Op(opAssignAppend) + vt.Values( + "${WRKSRC}", + "${WRKSRC}/", + "${WRKSRC}/.", + "${WRKSRC}/subdir", + "${CONFIGURE_DIRS}", + "${WRKSRC}/directory with spaces", + "directory with spaces") + vt.Output( + "NOTE: fname:1: You can use \".\" instead of \"${WRKSRC}\".", + "NOTE: fname:2: You can use \".\" instead of \"${WRKSRC}/\".", + "NOTE: fname:3: You can use \".\" instead of \"${WRKSRC}/.\".", + "NOTE: fname:4: You can use \"subdir\" instead of \"${WRKSRC}/subdir\".", + "NOTE: fname:6: You can use \"directory with spaces\" instead of \"${WRKSRC}/directory with spaces\".", + "WARN: fname:7: \"directory with spaces\" is not a valid subdirectory of ${WRKSRC}.") } func (s *Suite) Test_VartypeCheck_Yes(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Yes) - runVartypeChecks(t, "APACHE_MODULE", opAssign, (*VartypeCheck).Yes, + vt.Varname("APACHE_MODULE") + vt.Values( "yes", "no", "${YESVAR}") - t.CheckOutputLines( + vt.Output( "WARN: fname:2: APACHE_MODULE should be set to YES or yes.", "WARN: fname:3: APACHE_MODULE should be set to YES or yes.") - runVartypeMatchChecks(t, "PKG_DEVELOPER", (*VartypeCheck).Yes, + vt.Varname("PKG_DEVELOPER") + vt.Op(opUseMatch) + vt.Values( "yes", "no", "${YESVAR}") - t.CheckOutputLines( - "WARN: fname:1: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", - "WARN: fname:2: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", - "WARN: fname:3: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.") + vt.Output( + "WARN: fname:11: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", + "WARN: fname:12: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", + "WARN: fname:13: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.") } func (s *Suite) Test_VartypeCheck_YesNo(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).YesNo) - runVartypeChecks(t, "GNU_CONFIGURE", opAssign, (*VartypeCheck).YesNo, + vt.Varname("GNU_CONFIGURE") + vt.Values( "yes", "no", "ja", "${YESVAR}") - t.CheckOutputLines( + vt.Output( "WARN: fname:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.", "WARN: fname:4: GNU_CONFIGURE should be set to YES, yes, NO, or no.") } func (s *Suite) Test_VartypeCheck_YesNoIndirectly(c *check.C) { - t := s.Init(c) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).YesNoIndirectly) - runVartypeChecks(t, "GNU_CONFIGURE", opAssign, (*VartypeCheck).YesNoIndirectly, + vt.Varname("GNU_CONFIGURE") + vt.Values( "yes", "no", "ja", "${YESVAR}") - t.CheckOutputLines( + vt.Output( "WARN: fname:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.") } -func runVartypeChecks(t *Tester, varname string, op MkOperator, checker func(*VartypeCheck), values ...string) { - if !contains(op.String(), "=") { - panic("runVartypeChecks needs an assignment operator") - } - for i, value := range values { - mkline := t.NewMkLine("fname", i+1, varname+op.String()+value) - mkline.Tokenize(mkline.Value()) - valueNovar := mkline.WithoutMakeVariables(mkline.Value()) - vc := &VartypeCheck{mkline, mkline.Line, mkline.Varname(), mkline.Op(), mkline.Value(), valueNovar, "", false} - checker(vc) - } +// VartypeCheckTester helps to test the many different checks in VartypeCheck. +// It keeps track of the current variable, operator, file name, line number, +// so that the test can focus on the interesting details. +type VartypeCheckTester struct { + tester *Tester + checker func(cv *VartypeCheck) + fileName string + lineno int + varname string + op MkOperator +} + +// NewVartypeCheckTester starts the test with a file name of "fname", at line 1, +// with "=" as the operator. The variable has to be initialized explicitly. +func NewVartypeCheckTester(t *Tester, checker func(cv *VartypeCheck)) *VartypeCheckTester { + return &VartypeCheckTester{ + t, + checker, + "fname", + 1, + "", + opAssign} } -func runVartypeMatchChecks(t *Tester, varname string, checker func(*VartypeCheck), values ...string) { - for i, value := range values { - text := fmt.Sprintf(".if ${%s:M%s} == \"\"", varname, value) - mkline := t.NewMkLine("fname", i+1, text) - valueNovar := mkline.WithoutMakeVariables(value) - vc := &VartypeCheck{mkline, mkline.Line, varname, opUseMatch, value, valueNovar, "", false} - checker(vc) +func (vt *VartypeCheckTester) Varname(varname string) { + vt.varname = varname + vt.nextSection() +} + +func (vt *VartypeCheckTester) File(fileName string) { + vt.fileName = fileName + vt.lineno = 1 +} + +// Op sets the operator for the following tests. +// The line number is advanced to the next number ending in 1, e.g. 11, 21, 31. +func (vt *VartypeCheckTester) Op(op MkOperator) { + vt.op = op + vt.nextSection() +} + +// Values feeds each of the values to the actual check. +// Each value is interpreted as if it were written verbatim into a Makefile line. +// That is, # starts a comment, and for the opUseMatch operator, all closing braces must be escaped. +func (vt *VartypeCheckTester) Values(values ...string) { + for _, value := range values { + op := vt.op + opStr := op.String() + varname := vt.varname + + var text string + switch { + case contains(opStr, "="): + if hasSuffix(varname, "+") && opStr == "=" { + text = varname + " " + opStr + value + } else { + text = varname + opStr + value + } + case op == opUseMatch: + text = fmt.Sprintf(".if ${%s:M%s} == \"\"", varname, value) + default: + panic("Invalid operator: " + opStr) + } + + mkline := vt.tester.NewMkLine(vt.fileName, vt.lineno, text) + comment := "" + if mkline.IsVarassign() { + mkline.Tokenize(value, true) // Produce some warnings as side-effects. + comment = mkline.VarassignComment() + } + + effectiveValue := value + if mkline.IsVarassign() { + effectiveValue = mkline.Value() + } + + valueNovar := mkline.WithoutMakeVariables(effectiveValue) + vc := &VartypeCheck{mkline, mkline.Line, varname, op, effectiveValue, valueNovar, comment, false} + vt.checker(vc) + + vt.lineno++ } } -func runVartypeChecksFname(t *Tester, fname, varname string, op MkOperator, checker func(*VartypeCheck), values ...string) { - for i, value := range values { - mkline := t.NewMkLine(fname, i+1, varname+op.String()+value) - valueNovar := mkline.WithoutMakeVariables(value) - vc := &VartypeCheck{mkline, mkline.Line, varname, op, value, valueNovar, "", false} - checker(vc) +// Output checks that the output from all previous steps is +// the same as the expectedLines. +func (vt *VartypeCheckTester) Output(expectedLines ...string) { + vt.tester.CheckOutputLines(expectedLines...) +} + +func (vt *VartypeCheckTester) OutputEmpty() { + vt.tester.CheckOutputEmpty() +} + +func (vt *VartypeCheckTester) nextSection() { + if vt.lineno%10 != 1 { + vt.lineno = vt.lineno - vt.lineno%10 + 11 } } Index: pkgsrc/pkgtools/pkglint/files/mklines_test.go diff -u pkgsrc/pkgtools/pkglint/files/mklines_test.go:1.26 pkgsrc/pkgtools/pkglint/files/mklines_test.go:1.27 --- pkgsrc/pkgtools/pkglint/files/mklines_test.go:1.26 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/mklines_test.go Wed Sep 5 17:56:22 2018 @@ -86,9 +86,9 @@ func (s *Suite) Test_MkLines__for_loop_m t := s.Init(c) t.SetupCommandLine("-Wall") - t.SetupTool(&Tool{Name: "echo", Varname: "ECHO", Predefined: true}) - t.SetupTool(&Tool{Name: "find", Varname: "FIND", Predefined: true}) - t.SetupTool(&Tool{Name: "pax", Varname: "PAX", Predefined: true}) + t.SetupToolUsable("echo", "ECHO") + t.SetupToolUsable("find", "FIND") + t.SetupToolUsable("pax", "PAX") mklines := t.NewMkLines("Makefile", // From audio/squeezeboxserver MkRcsID, "", @@ -214,7 +214,7 @@ func (s *Suite) Test_MkLines__indirect_v "", "post-configure:", ".for var in MAIL_PROGRAM CMDPATH", - "\t"+`${RUN} ${ECHO} "#define ${var} \""${UUCP_${var}}"\"`, + "\t"+`${RUN} ${ECHO} "#define ${var} \""${UUCP_${var}}"\""`, ".endfor") mklines.Check() @@ -313,6 +313,49 @@ func (s *Suite) Test_MkLines_checkForUse c.Check(G.autofixAvailable, equals, true) } +func (s *Suite) Test_MkLines_DetermineDefinedVariables(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + t.SetupPkgsrc() + t.CreateFileLines("mk/tools/defaults.mk", + "USE_TOOLS+= autoconf autoconf213") + G.Pkgsrc.LoadInfrastructure() + mklines := t.NewMkLines("determine-defined-variables.mk", + MkRcsID, + "", + "USE_TOOLS+= autoconf213 autoconf", + "USE_TOOLS:= ${USE_TOOLS:Ntbl}", + "", + "OPSYSVARS+= OSV", + "OSV.NetBSD= NetBSD-specific value", + "", + "SUBST_CLASSES+= subst", + "SUBST_STAGE.subst= pre-configure", + "SUBST_FILES.subst= file", + "SUBST_VARS.subst= SUV", + "SUV= value for substitution", + "", + "pre-configure:", + "\t${RUN} autoreconf; autoheader-2.13; unknown-command", + "\t${ECHO} ${OSV:Q}") + + mklines.Check() + + // The tools autoreconf and autoheader213 are known at this point because of the USE_TOOLS line. + // The SUV variable is used implicitly by the SUBST framework, therefore no warning. + // The OSV.NetBSD variable is used implicitly via the OSV variable, therefore no warning. + t.CheckOutputLines( + // FIXME: For most lists, using the := operator to exclude an item is ok. + "WARN: determine-defined-variables.mk:4: USE_TOOLS should not be evaluated at load time.", + "WARN: determine-defined-variables.mk:4: USE_TOOLS may not be used in any file; it is a write-only variable.", + // FIXME: the below warning is wrong; it's ok to have SUBST blocks in all files, maybe except buildlink3.mk. + "WARN: determine-defined-variables.mk:12: The variable SUBST_VARS.subst may not be set (only given a default value, appended to) in this file; it would be ok in Makefile, Makefile.common, options.mk.", + // FIXME: the below warning is wrong; variables mentioned in SUBST_VARS should be allowed in that block. + "WARN: determine-defined-variables.mk:13: Foreign variable \"SUV\" in SUBST block.", + "WARN: determine-defined-variables.mk:16: Unknown shell command \"unknown-command\".") +} + func (s *Suite) Test_MkLines_DetermineUsedVariables__simple(c *check.C) { t := s.Init(c) @@ -372,7 +415,9 @@ func (s *Suite) Test_MkLines_PrivateTool mklines.Check() - t.CheckOutputEmpty() + // TODO: Is it necessary to add the tool to USE_TOOLS? If not, why not? + t.CheckOutputLines( + "WARN: fname:4: The \"md5sum\" tool is used but not added to USE_TOOLS.") } func (s *Suite) Test_MkLines_Check_indentation(c *check.C) { @@ -398,7 +443,7 @@ func (s *Suite) Test_MkLines_Check_inden mklines.Check() - t.CheckOutputLines(""+ + t.CheckOutputLines( "NOTE: options.mk:2: This directive should be indented by 0 spaces.", "NOTE: options.mk:3: This directive should be indented by 0 spaces.", "NOTE: options.mk:4: This directive should be indented by 2 spaces.", @@ -446,13 +491,31 @@ func (s *Suite) Test_MkLines_Check__endi // See MkLineChecker.checkDirective mklines.Check() - t.CheckOutputLines(""+ + t.CheckOutputLines( "WARN: opsys.mk:7: Comment \"ARCH\" does not match condition \"${OS_VERSION:M8.*}\".", "WARN: opsys.mk:8: Comment \"OS_VERSION\" does not match condition \"${ARCH} == x86_64\".", "WARN: opsys.mk:10: Comment \"j\" does not match loop \"i in 1 2 3 4 5\".", "WARN: opsys.mk:20: Comment \"NetBSD\" does not match condition \"${OPSYS} == FreeBSD\".") } +func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + mklines := t.NewMkLines("opsys.mk", + MkRcsID, + "", + ".for i in 1 2 3 4 5", + ". if ${OPSYS} == NetBSD", + ". if ${ARCH} == x86_64", + ". if ${OS_VERSION:M8.*}") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: opsys.mk:6: Directive indentation is not 0, but 8.") +} + // Demonstrates how to define your own make(1) targets for creating // files in the current directory. The pkgsrc-wip category Makefile // does this, while all other categories don't need any custom code. @@ -461,7 +524,7 @@ func (s *Suite) Test_MkLines_wip_categor t.SetupCommandLine("-Wall", "--explain") t.SetupVartypes() - t.SetupTool(&Tool{Name: "rm", Varname: "RM", Predefined: true}) + t.SetupToolUsable("rm", "RM") t.CreateFileLines("mk/misc/category.mk") mklines := t.SetupFileMkLines("wip/Makefile", MkRcsID, @@ -498,15 +561,19 @@ func (s *Suite) Test_MkLines_wip_categor "") } -func (s *Suite) Test_MkLines_ExtractDocumentedVariables(c *check.C) { +func (s *Suite) Test_MkLines_determineDocumentedVariables(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") t.SetupVartypes() - t.SetupTool(&Tool{Name: "rm", Varname: "RM", Predefined: true}) + t.SetupToolUsable("rm", "RM") mklines := t.NewMkLines("Makefile", MkRcsID, "#", + "# Copyright 2000-2018", + "#", + "# This whole comment is ignored, until the next empty line.", + "", "# User-settable variables:", "#", "# PKG_DEBUG_LEVEL", @@ -536,12 +603,12 @@ func (s *Suite) Test_MkLines_ExtractDocu sort.Strings(varnames) expected := []string{ - "PKG_DEBUG_LEVEL (line 5)", - "PKG_VERBOSE (line 10)", - "VARBASE1.* (line 17)", - "VARBASE2.* (line 18)", - "VARBASE3.${id} (line 19)", - "VARBASE3.* (line 19)"} + "PKG_DEBUG_LEVEL (line 9)", + "PKG_VERBOSE (line 14)", + "VARBASE1.* (line 21)", + "VARBASE2.* (line 22)", + "VARBASE3.${id} (line 23)", + "VARBASE3.* (line 23)"} c.Check(varnames, deepEquals, expected) } @@ -639,6 +706,34 @@ func (s *Suite) Test_MkLines_CheckRedund t.CheckOutputEmpty() } +func (s *Suite) Test_MkLines_CheckRedundantVariables__shell_and_eval(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR:=\tvalue ${OTHER}", + "VAR!=\tvalue ${OTHER}") + + mklines.CheckRedundantVariables() + + // Combining := and != is too complicated to be analyzed by pkglint, + // therefore no warning. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLines_CheckRedundantVariables__shell_and_eval_literal(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR:=\tvalue", + "VAR!=\tvalue") + + mklines.CheckRedundantVariables() + + // Even when := is used with a literal value (which is usually + // only done for procedure calls), the shell evaluation can have + // so many different side effects that pkglint cannot reliably + // help in this situation. + t.CheckOutputEmpty() +} + func (s *Suite) Test_MkLines_Check__PLIST_VARS(c *check.C) { t := s.Init(c) @@ -785,3 +880,19 @@ func (s *Suite) Test_MkLines_Check__indi // Therefore, in such a case, no diagnostics are logged at all. t.CheckOutputEmpty() } + +func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + t.SetupVartypes() + mklines := t.NewMkLines("hacks.mk", + MkRcsID, + "", + "PKGNAME?= pkgbase-1.0") + + mklines.Check() + + // No warning about including bsd.prefs.mk before using the ?= operator. + t.CheckOutputEmpty() +} Index: pkgsrc/pkgtools/pkglint/files/mkparser.go diff -u pkgsrc/pkgtools/pkglint/files/mkparser.go:1.15 pkgsrc/pkgtools/pkglint/files/mkparser.go:1.16 --- pkgsrc/pkgtools/pkglint/files/mkparser.go:1.15 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/mkparser.go Wed Sep 5 17:56:22 2018 @@ -134,9 +134,10 @@ loop: case '=', 'D', 'M', 'N', 'U': if repl.AdvanceRegexp(`^[=DMNU]`) { - for p.VarUse() != nil || repl.AdvanceRegexp(regex.Pattern(`^([^$:`+closing+`]|\$\$)+`)) { + for p.VarUse() != nil || repl.AdvanceRegexp(regex.Pattern(`^([^$:\\`+closing+`]|\$\$|\\.)+`)) { } - modifiers = append(modifiers, repl.Since(modifierMark)) + arg := repl.Since(modifierMark) + modifiers = append(modifiers, strings.Replace(arg, "\\:", ":", -1)) continue } Index: pkgsrc/pkgtools/pkglint/files/vartype.go diff -u pkgsrc/pkgtools/pkglint/files/vartype.go:1.15 pkgsrc/pkgtools/pkglint/files/vartype.go:1.16 --- pkgsrc/pkgtools/pkglint/files/vartype.go:1.15 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/vartype.go Wed Sep 5 17:56:22 2018 @@ -79,9 +79,9 @@ func (vt *Vartype) EffectivePermissions( return aclpUnknown } -// Returns the union of all possible permissions. This can be used to -// check whether a variable may be defined or used at all, or if it is -// read-only. +// Union returns the union of all possible permissions. +// This can be used to check whether a variable may be defined or used +// at all, or if it is read-only. func (vt *Vartype) Union() ACLPermissions { var permissions ACLPermissions for _, aclEntry := range vt.aclEntries { @@ -100,7 +100,7 @@ func (vt *Vartype) AllowedFiles(perms AC return strings.Join(files, ", ") } -// Returns whether the type is considered a shell list. +// IsConsideredList returns whether the type is considered a shell list. // This distinction between "real lists" and "considered a list" makes // the implementation of checklineMkVartype easier. func (vt *Vartype) IsConsideredList() bool { @@ -122,20 +122,8 @@ func (vt *Vartype) MayBeAppendedTo() boo } func (vt *Vartype) String() string { - listPrefix := "" - switch vt.kindOfList { - case lkNone: - listPrefix = "" - case lkSpace: - listPrefix = "SpaceList of " - case lkShell: - listPrefix = "ShellList of " - default: - panic("Unknown list type") - } - + listPrefix := [...]string{"", "SpaceList of ", "ShellList of "}[vt.kindOfList] guessedSuffix := ifelseStr(vt.guessed, " (guessed)", "") - return listPrefix + vt.basicType.name + guessedSuffix } Index: pkgsrc/pkgtools/pkglint/files/mkshparser_test.go diff -u pkgsrc/pkgtools/pkglint/files/mkshparser_test.go:1.5 pkgsrc/pkgtools/pkglint/files/mkshparser_test.go:1.6 --- pkgsrc/pkgtools/pkglint/files/mkshparser_test.go:1.5 Sun Jan 1 15:15:47 2017 +++ pkgsrc/pkgtools/pkglint/files/mkshparser_test.go Wed Sep 5 17:56:22 2018 @@ -6,6 +6,21 @@ import ( "strconv" ) +func (s *Suite) Test_parseShellProgram__parse_error(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("module.mk", 1, "\t$${") + + list, err := parseShellProgram(mkline.Line, mkline.ShellCommand()) + + c.Check(list, check.IsNil) + // XXX: []string{"$${"} would be an even better error message + c.Check(err.Error(), equals, "parse error at []string{\"\"}") + + t.CheckOutputLines( + "WARN: module.mk:1: Pkglint parse error in ShTokenizer.ShAtom at \"$${\" (quoting=plain).") +} + type ShSuite struct { c *check.C } @@ -79,6 +94,13 @@ func (s *ShSuite) Test_ShellParser_progr b.CaseItem( b.Words("pattern"), b.List().AddCommand(b.SimpleCommand("case-item-action")), sepNone))).AddSemicolon()))) + + s.test("if condition; then action; elif condition2; then action2; fi", + b.List().AddCommand(b.If( + b.List().AddCommand(b.SimpleCommand("condition")).AddSemicolon(), + b.List().AddCommand(b.SimpleCommand("action")).AddSemicolon(), + b.List().AddCommand(b.SimpleCommand("condition2")).AddSemicolon(), + b.List().AddCommand(b.SimpleCommand("action2")).AddSemicolon()))) } func (s *ShSuite) Test_ShellParser_list(c *check.C) { @@ -489,20 +511,14 @@ func (s *ShSuite) test(program string, e } func (s *ShSuite) Test_ShellLexer_Lex__redirects(c *check.C) { - tokens, rest := splitIntoShellTokens(dummyLine, "${MAKE} print-summary-data 2>&1 > /dev/stderr") + tokens, rest := splitIntoShellTokens(dummyLine, "2>&1 <& <>file >>file < /dev/stderr") - c.Check(tokens, deepEquals, []string{"${MAKE}", "print-summary-data", "2>&", "1", ">", "/dev/stderr"}) + c.Check(tokens, deepEquals, []string{"2>&", "1", "<&", "<>", "file", ">>", "file", "<<", "EOF", "<<-", "EOF", ">", "/dev/stderr"}) c.Check(rest, equals, "") lexer := NewShellLexer(tokens, rest) var llval shyySymType - c.Check(lexer.Lex(&llval), equals, tkWORD) - c.Check(llval.Word.MkText, equals, "${MAKE}") - - c.Check(lexer.Lex(&llval), equals, tkWORD) - c.Check(llval.Word.MkText, equals, "print-summary-data") - c.Check(lexer.Lex(&llval), equals, tkIO_NUMBER) c.Check(llval.IONum, equals, 2) @@ -511,6 +527,28 @@ func (s *ShSuite) Test_ShellLexer_Lex__r c.Check(lexer.Lex(&llval), equals, tkWORD) c.Check(llval.Word.MkText, equals, "1") + c.Check(lexer.Lex(&llval), equals, tkLTAND) + + c.Check(lexer.Lex(&llval), equals, tkLTGT) + + c.Check(lexer.Lex(&llval), equals, tkWORD) + c.Check(llval.Word.MkText, equals, "file") + + c.Check(lexer.Lex(&llval), equals, tkGTGT) + + c.Check(lexer.Lex(&llval), equals, tkWORD) + c.Check(llval.Word.MkText, equals, "file") + + c.Check(lexer.Lex(&llval), equals, tkLTLT) + + c.Check(lexer.Lex(&llval), equals, tkWORD) + c.Check(llval.Word.MkText, equals, "EOF") + + c.Check(lexer.Lex(&llval), equals, tkLTLTDASH) + + c.Check(lexer.Lex(&llval), equals, tkWORD) + c.Check(llval.Word.MkText, equals, "EOF") + c.Check(lexer.Lex(&llval), equals, tkGT) c.Check(lexer.Lex(&llval), equals, tkWORD) Index: pkgsrc/pkgtools/pkglint/files/package.go diff -u pkgsrc/pkgtools/pkglint/files/package.go:1.34 pkgsrc/pkgtools/pkglint/files/package.go:1.35 --- pkgsrc/pkgtools/pkglint/files/package.go:1.34 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/package.go Wed Sep 5 17:56:22 2018 @@ -1,6 +1,7 @@ package main import ( + "fmt" "netbsd.org/pkglint/pkgver" "netbsd.org/pkglint/regex" "netbsd.org/pkglint/trace" @@ -25,7 +26,6 @@ type Package struct { EffectivePkgbase string // The effective PKGNAME without the version EffectivePkgversion string // The version part of the effective PKGNAME, excluding nb13 EffectivePkgnameLine MkLine // The origin of the three effective_* values - SeenBsdPrefsMk bool // Has bsd.prefs.mk already been included? PlistDirs map[string]bool // Directories mentioned in the PLIST files PlistFiles map[string]bool // Regular files mentioned in the PLIST files @@ -34,7 +34,6 @@ type Package struct { plistSubstCond map[string]bool // varname => true; all variables that are used as conditions (@comment or nothing) in PLISTs. included map[string]Line // fname => line seenMakefileCommon bool // Does the package have any .includes? - loadTimeTools map[string]bool // true=ok, false=not ok, absent=not mentioned in USE_TOOLS. conditionalIncludes map[string]MkLine unconditionalIncludes map[string]MkLine once Once @@ -44,7 +43,7 @@ type Package struct { func NewPackage(dir string) *Package { pkgpath := G.Pkgsrc.ToRel(dir) if strings.Count(pkgpath, "/") != 1 { - NewLineWhole(dir).Errorf("Package directory %q must be two subdirectories below the pkgsrc root %q.", dir, G.Pkgsrc.File(".")) + panic(fmt.Sprintf("Package directory %q must be two subdirectories below the pkgsrc root %q.", dir, G.Pkgsrc.File("."))) } pkg := &Package{ @@ -60,13 +59,10 @@ func NewPackage(dir string) *Package { bl3: make(map[string]Line), plistSubstCond: make(map[string]bool), included: make(map[string]Line), - loadTimeTools: make(map[string]bool), conditionalIncludes: make(map[string]MkLine), unconditionalIncludes: make(map[string]MkLine), } - for varname, line := range G.Pkgsrc.UserDefinedVars { - pkg.vars.Define(varname, line) - } + pkg.vars.DefineAll(G.Pkgsrc.UserDefinedVars) return pkg } @@ -76,28 +72,6 @@ func (pkg *Package) File(relativeFilenam return cleanpath(pkg.dir + "/" + relativeFilename) } -func (pkg *Package) varValue(varname string) (string, bool) { - switch varname { - case "KRB5_TYPE": - return "heimdal", true - case "PGSQL_VERSION": - return "95", true - } - if mkline := pkg.vars.FirstDefinition(varname); mkline != nil { - return mkline.Value(), true - } - return "", false -} - -func (pkg *Package) setSeenBsdPrefsMk() { - if !pkg.SeenBsdPrefsMk { - pkg.SeenBsdPrefsMk = true - if trace.Tracing { - trace.Stepf("Pkg.setSeenBsdPrefsMk") - } - } -} - func (pkg *Package) checkPossibleDowngrade() { if trace.Tracing { defer trace.Call0()() @@ -357,7 +331,7 @@ func (pkg *Package) readMakefile(fname s G.Pkg.seenMakefileCommon = true } - skip := contains(fname, "/mk/") || hasSuffix(includeFile, "/bsd.pkg.mk") || hasSuffix(includeFile, "/bsd.prefs.mk") + skip := contains(fname, "/mk/") || hasSuffix(includeFile, "/bsd.pkg.mk") || IsPrefs(includeFile) if !skip { dirname, _ := path.Split(fname) dirname = cleanpath(dirname) @@ -405,7 +379,7 @@ func (pkg *Package) readMakefile(fname s return true } atEnd := func(mkline MkLine) {} - fileMklines.ForEach(lineAction, atEnd) + fileMklines.ForEachEnd(lineAction, atEnd) if includingFnameForUsedCheck != "" { fileMklines.checkForUsedComment(G.Pkgsrc.ToRel(includingFnameForUsedCheck)) @@ -481,11 +455,7 @@ func (pkg *Package) checkfilePackageMake } func (pkg *Package) getNbpart() string { - line := pkg.vars.FirstDefinition("PKGREVISION") - if line == nil { - return "" - } - pkgrevision := line.Value() + pkgrevision, _ := pkg.vars.Value("PKGREVISION") if rev, err := strconv.Atoi(pkgrevision); err == nil { return "nb" + strconv.Itoa(rev) } @@ -546,7 +516,7 @@ func (pkg *Package) pkgnameFromDistname( subst := func(str, smod string) (result string) { if trace.Tracing { - defer trace.Call(str, smod, trace.Ref(result))() + defer trace.Call(str, smod, trace.Result(&result))() } qsep := regexp.QuoteMeta(smod[1:2]) if m, left, from, right, to, flags := regex.Match5(smod, regex.Pattern(`^S`+qsep+`(\^?)([^:]*?)(\$?)`+qsep+`([^:]*)`+qsep+`([1g]*)$`)); m { @@ -876,15 +846,13 @@ func (pkg *Package) checkLocallyModified defer trace.Call(fname)() } - ownerLine := pkg.vars.FirstDefinition("OWNER") - maintainerLine := pkg.vars.FirstDefinition("MAINTAINER") - owner := "" - maintainer := "" - if ownerLine != nil && !containsVarRef(ownerLine.Value()) { - owner = ownerLine.Value() + owner, _ := pkg.vars.Value("OWNER") + maintainer, _ := pkg.vars.Value("MAINTAINER") + if containsVarRef(owner) { + owner = "" } - if maintainerLine != nil && !containsVarRef(maintainerLine.Value()) && maintainerLine.Value() != "pkgsrc-users@NetBSD.org" { - maintainer = maintainerLine.Value() + if containsVarRef(maintainer) || maintainer == "pkgsrc-users@NetBSD.org" { + maintainer = "" } if owner == "" && maintainer == "" { return Index: pkgsrc/pkgtools/pkglint/files/package_test.go diff -u pkgsrc/pkgtools/pkglint/files/package_test.go:1.28 pkgsrc/pkgtools/pkglint/files/package_test.go:1.29 --- pkgsrc/pkgtools/pkglint/files/package_test.go:1.28 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/package_test.go Wed Sep 5 17:56:22 2018 @@ -336,8 +336,10 @@ func (s *Suite) Test_Package__varuse_at_ t := s.Init(c) t.SetupPkgsrc() - t.CreateFileLines("licenses/bsd-2", + t.SetupToolUsable("printf", "") + t.CreateFileLines("licenses/2-clause-bsd", "# dummy") + t.CreateFileLines("misc/Makefile") t.CreateFileLines("mk/tools/defaults.mk", "TOOLS_CREATE+=false", "TOOLS_CREATE+=nice", @@ -347,39 +349,66 @@ func (s *Suite) Test_Package__varuse_at_ t.CreateFileLines("category/pkgbase/Makefile", MkRcsID, "", - "COMMENT= Unit test", - "LICENSE= bsd-2", - "PLIST_SRC=#none", - "", - "USE_TOOLS+= echo false", - "FALSE_BEFORE!= echo false=${FALSE}", - "NICE_BEFORE!= echo nice=${NICE}", - "TRUE_BEFORE!= echo true=${TRUE}", + "PKGNAME= loadtime-vartest-1.0", + "CATEGORIES= misc", "", - ".include \"../../mk/bsd.prefs.mk\"", + "COMMENT= Demonstrate variable values during parsing", + "LICENSE= 2-clause-bsd", + "", + "PLIST_SRC= # none", + "NO_CHECKSUM= yes", + "NO_CONFIGURE= yes", + "", + "USE_TOOLS+= echo false", + "FALSE_BEFORE!= echo false=${FALSE:Q}", // false= + "NICE_BEFORE!= echo nice=${NICE:Q}", // nice= + "TRUE_BEFORE!= echo true=${TRUE:Q}", // true= + // + // All three variables above are empty since the tool + // variables are initialized by bsd.prefs.mk. The variables + // from share/mk/sys.mk are available, though. + // "", - "USE_TOOLS+= nice", - "FALSE_AFTER!= echo false=${FALSE}", - "NICE_AFTER!= echo nice=${NICE}", - "TRUE_AFTER!= echo true=${TRUE}", + ".include \"../../mk/bsd.prefs.mk\"", + // + // Now all tools from USE_TOOLS are defined with their variables. + // ${FALSE} works, but a plain "false" might call the wrong tool. + // That's because the tool wrappers are not set up yet. This + // happens between the post-depends and pre-fetch stages. Even + // then, the plain tool names may only be used in the + // {pre,do,post}-* targets, since a recursive make(1) needs to be + // run to set up the correct PATH. + // + "", + "USE_TOOLS+= nice", + // + // The "nice" tool will only be available as ${NICE} after bsd.pkg.mk + // has been included. Even including bsd.prefs.mk another time does + // not have any effect since it is guarded against multiple inclusion. + // + "", + ".include \"../../mk/bsd.prefs.mk\"", // Has no effect. + "", + "FALSE_AFTER!= echo false=${FALSE:Q}", // false=false + "NICE_AFTER!= echo nice=${NICE:Q}", // nice= + "TRUE_AFTER!= echo true=${TRUE:Q}", // true=true "", "do-build:", - "\t${ECHO} before: ${FALSE_BEFORE} ${NICE_BEFORE} ${TRUE_BEFORE}", - "\t${ECHO} after: ${FALSE_AFTER} ${NICE_AFTER} ${TRUE_AFTER}", - "\t${ECHO}; ${FALSE}; ${NICE}; ${TRUE}", + "\t${RUN} printf 'before: %-20s %-20s %-20s\\n' ${FALSE_BEFORE} ${NICE_BEFORE} ${TRUE_BEFORE}", + "\t${RUN} printf 'after: %-20s %-20s %-20s\\n' ${FALSE_AFTER} ${NICE_AFTER} ${TRUE_AFTER}", + "\t${RUN} printf 'runtime: %-20s %-20s %-20s\\n' false=${FALSE:Q} nice=${NICE:Q} true=${TRUE:Q}", "", ".include \"../../mk/bsd.pkg.mk\"") - t.CreateFileLines("category/pkgbase/distinfo", - RcsID) - G.Main("pkglint", "-q", "-Wperm", t.File("category/pkgbase")) + t.SetupCommandLine("-q", "-Wall,no-space") + G.Pkgsrc.LoadInfrastructure() + G.CheckDirent(t.File("category/pkgbase")) t.CheckOutputLines( - "WARN: ~/category/pkgbase/Makefile:8: To use the tool \"FALSE\" at load time, bsd.prefs.mk has to be included before.", - "WARN: ~/category/pkgbase/Makefile:9: To use the tool \"NICE\" at load time, bsd.prefs.mk has to be included before.", - "WARN: ~/category/pkgbase/Makefile:10: To use the tool \"TRUE\" at load time, bsd.prefs.mk has to be included before.", - "WARN: ~/category/pkgbase/Makefile:16: To use the tool \"NICE\" at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.", - "WARN: ~/category/pkgbase/Makefile:3: The canonical order of the variables is CATEGORIES, empty line, COMMENT, LICENSE.") + "WARN: ~/category/pkgbase/Makefile:14: To use the tool ${FALSE} at load time, bsd.prefs.mk has to be included before.", + "WARN: ~/category/pkgbase/Makefile:15: To use the tool ${NICE} at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.", + "WARN: ~/category/pkgbase/Makefile:16: To use the tool ${TRUE} at load time, bsd.prefs.mk has to be included before.", + "WARN: ~/category/pkgbase/Makefile:25: To use the tool ${NICE} at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.") } func (s *Suite) Test_Package_loadPackageMakefile(c *check.C) { @@ -550,3 +579,74 @@ func (s *Suite) Test_Package__redundant_ t.CheckOutputLines( "NOTE: ~/math/R-date/Makefile:6: Definition of MASTER_SITES is redundant because of ../R/Makefile.extension:4.") } + +func (s *Suite) Test_Package_checkUpdate(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.CreateFileLines("doc/TODO", + "Suggested package updates", + "", + "", + "\t"+"O wrong bullet", + "\t"+"o package-without-version", + "\t"+"o package1-1.0", + "\t"+"o package2-2.0 [nice new features]", + "\t"+"o package3-3.0 [security update]") + t.CreateFileLines("licenses/gnu-gpl-v2", + "The licenses for most software are designed to take away ...") + + t.CreateFileLines("category/pkg1/Makefile", + MkRcsID, + "", + "PKGNAME= package1-1.0", + "GENERATE_PLIST+= echo \"bin/program\";", + "NO_CHECKSUM= yes", + "LICENSE= gnu-gpl-v2") + t.CreateFileLines("category/pkg2/Makefile", + MkRcsID, + "", + "PKGNAME= package2-1.0", + "GENERATE_PLIST+= echo \"bin/program\";", + "NO_CHECKSUM= yes", + "LICENSE= gnu-gpl-v2") + t.CreateFileLines("category/pkg3/Makefile", + MkRcsID, + "", + "PKGNAME= package3-5.0", + "GENERATE_PLIST+= echo \"bin/program\";", + "NO_CHECKSUM= yes", + "LICENSE= gnu-gpl-v2") + + t.Chdir(".") + G.Main("pkglint", "-Wall,no-space,no-order", "category/pkg1", "category/pkg2", "category/pkg3") + + t.CheckOutputLines( + "WARN: category/pkg1/../../doc/TODO:3: Invalid line format \"\".", + "WARN: category/pkg1/../../doc/TODO:4: Invalid line format \"\\tO wrong bullet\".", + "WARN: category/pkg1/../../doc/TODO:5: Invalid package name \"package-without-version\".", + "WARN: category/pkg1/Makefile: No COMMENT given.", + "NOTE: category/pkg1/Makefile:3: The update request to 1.0 from doc/TODO has been done.", + "WARN: category/pkg1/Makefile:4: Please use \"${ECHO}\" instead of \"echo\".", + "WARN: category/pkg2/Makefile: No COMMENT given.", + "WARN: category/pkg2/Makefile:3: This package should be updated to 2.0 ([nice new features]).", + "WARN: category/pkg2/Makefile:4: Please use \"${ECHO}\" instead of \"echo\".", + "WARN: category/pkg3/Makefile: No COMMENT given.", + "NOTE: category/pkg3/Makefile:3: This package is newer than the update request to 3.0 ([security update]).", + "WARN: category/pkg3/Makefile:4: Please use \"${ECHO}\" instead of \"echo\".", + "0 errors and 10 warnings found.", + "(Run \"pkglint -e\" to show explanations.)") +} + +func (s *Suite) Test_NewPackage(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.CreateFileLines("category/Makefile", + MkRcsID) + + c.Check( + func() { NewPackage("category") }, + check.PanicMatches, + `Package directory "category" must be two subdirectories below the pkgsrc root ".*".`) +} Index: pkgsrc/pkgtools/pkglint/files/patches.go diff -u pkgsrc/pkgtools/pkglint/files/patches.go:1.22 pkgsrc/pkgtools/pkglint/files/patches.go:1.23 --- pkgsrc/pkgtools/pkglint/files/patches.go:1.22 Sat Jul 28 18:31:23 2018 +++ pkgsrc/pkgtools/pkglint/files/patches.go Wed Sep 5 17:56:22 2018 @@ -117,6 +117,7 @@ func (ck *PatchChecker) checkUnifiedDiff hasHunks := false for ck.exp.AdvanceIfMatches(rePatchUniHunk) { + text := ck.exp.m[0] hasHunks = true linesToDel := toInt(ck.exp.Group(2), 1) linesToAdd := toInt(ck.exp.Group(4), 1) @@ -124,6 +125,7 @@ func (ck *PatchChecker) checkUnifiedDiff trace.Stepf("hunk -%d +%d", linesToDel, linesToAdd) } ck.checktextUniHunkCr() + ck.checktextRcsid(text) for !ck.exp.EOF() && (linesToDel > 0 || linesToAdd > 0 || hasPrefix(ck.exp.CurrentLine().Text, "\\")) { line := ck.exp.CurrentLine() @@ -145,7 +147,7 @@ func (ck *PatchChecker) checkUnifiedDiff case hasPrefix(text, "\\"): // \ No newline at end of file (or a translation of that message) default: - line.Errorf("Invalid line in unified patch hunk") + line.Errorf("Invalid line in unified patch hunk: %s", text) return } } @@ -234,9 +236,9 @@ func (ck *PatchChecker) checklineAdded(a case ftShell, ftIgnore: break case ftMakefile: - checklineOtherAbsolutePathname(line, addedText) + ck.checklineOtherAbsolutePathname(line, addedText) case ftSource: - checklineSourceAbsolutePathname(line, addedText) + ck.checklineSourceAbsolutePathname(line, addedText) case ftConfigure: if hasSuffix(addedText, ": Avoid regenerating within pkgsrc") { line.Errorf("This code must not be included in patches.") @@ -247,7 +249,7 @@ func (ck *PatchChecker) checklineAdded(a "mk/configure/gnu-configure.mk.") } default: - checklineOtherAbsolutePathname(line, addedText) + ck.checklineOtherAbsolutePathname(line, addedText) } } @@ -315,7 +317,7 @@ func (ft FileType) String() string { // This is used to select the proper subroutine for detecting absolute pathnames. func guessFileType(fname string) (fileType FileType) { if trace.Tracing { - defer trace.Call(fname, "=>", &fileType)() + defer trace.Call(fname, trace.Result(&fileType))() } basename := path.Base(fname) @@ -347,7 +349,7 @@ func guessFileType(fname string) (fileTy } // Looks for strings like "/dev/cd0" appearing in source code -func checklineSourceAbsolutePathname(line Line, text string) { +func (ck *PatchChecker) checklineSourceAbsolutePathname(line Line, text string) { if !strings.ContainsAny(text, "\"'") { return } @@ -369,7 +371,7 @@ func checklineSourceAbsolutePathname(lin } } -func checklineOtherAbsolutePathname(line Line, text string) { +func (ck *PatchChecker) checklineOtherAbsolutePathname(line Line, text string) { if trace.Tracing { defer trace.Call1(text)() } @@ -379,12 +381,18 @@ func checklineOtherAbsolutePathname(line } else if m, before, path, _ := match3(text, `^(.*?)((?:/[\w.]+)*/(?:bin|dev|etc|home|lib|mnt|opt|proc|sbin|tmp|usr|var)\b[\w./\-]*)(.*)$`); m { switch { - case hasSuffix(before, "@"): // Example: @PREFIX@/bin - case matches(before, `[)}]$`) && !matches(before, `DESTDIR[)}]$`): // Example: ${prefix}/bin - case matches(before, `\+\s*["']$`): // Example: prefix + '/lib' - case matches(before, `\$\w$`): // Example: libdir=$prefix/lib - case hasSuffix(before, "."): // Example: ../dir - // XXX new: case matches(before, `s.$`): // Example: sed -e s,/usr,@PREFIX@, + case matches(before, `[\w).@}]$`) && !matches(before, `DESTDIR.$`): + // Example: $prefix/bin + // Example: $(prefix)/bin + // Example: ../bin + // Example: @prefix@/bin + // Example: ${prefix}/bin + + case matches(before, `\+\s*["']$`): + // Example: prefix + '/lib' + + // XXX new: case matches(before, `\bs.$`): // Example: sed -e s,/usr,@PREFIX@, + default: if trace.Tracing { trace.Step1("before=%q", before) Index: pkgsrc/pkgtools/pkglint/files/patches_test.go diff -u pkgsrc/pkgtools/pkglint/files/patches_test.go:1.19 pkgsrc/pkgtools/pkglint/files/patches_test.go:1.20 --- pkgsrc/pkgtools/pkglint/files/patches_test.go:1.19 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/patches_test.go Wed Sep 5 17:56:22 2018 @@ -147,14 +147,49 @@ func (s *Suite) Test_ChecklinesPatch__gi "ERROR: patch-aa:5: Each patch must be documented.") } -func (s *Suite) Test_checklineOtherAbsolutePathname(c *check.C) { +func (s *Suite) Test_PatchChecker_checklineSourceAbsolutePathname(c *check.C) { t := s.Init(c) - line := t.NewLine("patch-ag", 1, "+$install -s -c ./bin/rosegarden ${DESTDIR}$BINDIR") + lines := t.NewLines("patch-aa", + RcsID, + "", + "Documentation", + "", + "--- code.c.orig", + "+++ code.c", + "@@ -0,0 +1,3 @@", + "+const char abspath[] = PREFIX \"/bin/program\";", + "+val abspath = libdir + \"/libiconv.so.1.0\"", + "+const char abspath[] = \"/dev/scd0\";", + ) - checklineOtherAbsolutePathname(line, line.Text) + ChecklinesPatch(lines) - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: patch-aa:10: Found absolute pathname: /dev/scd0") +} + +func (s *Suite) Test_PatchChecker_checklineOtherAbsolutePathname(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("patch-aa", + RcsID, + "", + "Documentation", + "", + "--- file.unknown.orig", + "+++ file.unknown", + "@@ -0,0 +1,5 @@", + "+abspath=\"@prefix@/bin/program\"", + "+abspath=\"${DESTDIR}/bin/\"", + "+abspath=\"${PREFIX}/bin/\"", + "+abspath = $prefix + '/bin/program'", + "+abspath=\"$prefix/bin/program\"") + + ChecklinesPatch(lines) + + t.CheckOutputLines( + "WARN: patch-aa:9: Found absolute pathname: /bin/") } // The output of BSD Make typically contains "*** Error code". @@ -522,6 +557,145 @@ func (s *Suite) Test_ChecklinesPatch__au "ERROR: ~/patch-aa:9: This code must not be included in patches.") } +func (s *Suite) Test_ChecklinesPatch__empty_context_lines_in_hunk(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + lines := t.SetupFileLines("patch-aa", + RcsID, + "", + "Documentation", + "", + "--- configure.orig", + "+++ configure", + "@@ -1,3 +1,3 @@", + "", + "-old line", + "+new line") + + ChecklinesPatch(lines) + + // The first context line should start with a single space character, + // but that would mean trailing white-space, so it may be left out. + // The last context line is omitted completely because it would also + // have trailing white-space, and if that were removed, would be a + // trailing empty line. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_ChecklinesPatch__invalid_line_in_hunk(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + lines := t.SetupFileLines("patch-aa", + RcsID, + "", + "Documentation", + "", + "--- configure.orig", + "+++ configure", + "@@ -1,3 +1,3 @@", + "", + "-old line", + "<<<<<<<<", + "+new line") + + ChecklinesPatch(lines) + + // The first context line should start with a single space character, + // but that would mean trailing white-space, so it may be left out. + // The last context line is omitted completely because it would also + // have trailing white-space, and if that were removed, would be a + // trailing empty line. + t.CheckOutputLines( + "ERROR: ~/patch-aa:10: Invalid line in unified patch hunk: <<<<<<<<") +} + +func (s *Suite) Test_PatchChecker_checklineAdded__shell(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + lines := t.SetupFileLines("patch-aa", + RcsID, + "", + "Documentation", + "", + "--- configure.sh.orig", + "+++ configure.sh", + "@@ -1,1 +1,1 @@", + "-old line", + "+new line") + + ChecklinesPatch(lines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_PatchChecker_checklineAdded__text(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + lines := t.SetupFileLines("patch-aa", + RcsID, + "", + "Documentation", + "", + "--- configure.tex.orig", + "+++ configure.tex", + "@@ -1,1 +1,1 @@", + "-old line", + "+new line") + + ChecklinesPatch(lines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_PatchChecker_checklineAdded__unknown(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + lines := t.SetupFileLines("patch-aa", + RcsID, + "", + "Documentation", + "", + "--- configure.unknown.orig", + "+++ configure.unknown", + "@@ -1,1 +1,1 @@", + "-old line", + "+new line") + + ChecklinesPatch(lines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_PatchChecker_checktextRcsid(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + lines := t.SetupFileLines("patch-aa", + RcsID, + "", + "Documentation", + "", + "--- configure.sh.orig", + "+++ configure.sh", + "@@ -1,3 +1,3 @@ $"+"Id$", + " $"+"Id$", + "-old line", + "+new line", + " $Author: rillig $") + + ChecklinesPatch(lines) + + t.CheckOutputLines( + "WARN: ~/patch-aa:7: Found RCS tag \"$Id: patches_test.go,v 1.20 2018/09/05 17:56:22 rillig Exp $\". Please remove it.", + "WARN: ~/patch-aa:8: Found RCS tag \"$Id: patches_test.go,v 1.20 2018/09/05 17:56:22 rillig Exp $\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", + "WARN: ~/patch-aa:11: Found RCS tag \"$Author: rillig $\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".") +} + func (s *Suite) Test_FileType_String(c *check.C) { c.Check(ftUnknown.String(), equals, "unknown") } Index: pkgsrc/pkgtools/pkglint/files/pkglint_test.go diff -u pkgsrc/pkgtools/pkglint/files/pkglint_test.go:1.23 pkgsrc/pkgtools/pkglint/files/pkglint_test.go:1.24 --- pkgsrc/pkgtools/pkglint/files/pkglint_test.go:1.23 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/pkglint_test.go Wed Sep 5 17:56:22 2018 @@ -1,6 +1,7 @@ package main import ( + "io/ioutil" "strings" "gopkg.in/check.v1" @@ -224,12 +225,14 @@ func (s *Suite) Test_Pkglint_Main__compl t.CheckOutputLines( "WARN: ~/sysutils/checkperms/Makefile:3: This package should be updated to 1.13 ([supports more file formats]).", "ERROR: ~/sysutils/checkperms/Makefile:4: Invalid category \"tools\".", + "ERROR: ~/sysutils/checkperms/README: Packages in main pkgsrc must not have a README file.", + "ERROR: ~/sysutils/checkperms/TODO: Packages in main pkgsrc must not have a TODO file.", "ERROR: ~/sysutils/checkperms/distinfo:7: SHA1 hash of patches/patch-checkperms.c differs "+ "(distinfo has asdfasdf, patch file has e775969de639ec703866c0336c4c8e0fdd96309c). "+ "Run \""+confMake+" makepatchsum\".", "WARN: ~/sysutils/checkperms/patches/patch-checkperms.c:12: Premature end of patch hunk "+ "(expected 1 lines to be deleted and 0 lines to be added).", - "2 errors and 2 warnings found.", + "4 errors and 2 warnings found.", "(Run \"pkglint -e\" to show explanations.)", "(Run \"pkglint -fs\" to show what can be fixed automatically.)", "(Run \"pkglint -F\" to automatically fix some issues.)") @@ -410,64 +413,6 @@ func (s *Suite) Test_ChecklinesMessage__ "===========================================================================") } -func (s *Suite) Test_Pkgsrc_Latest_no_basedir(c *check.C) { - t := s.Init(c) - - latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") - - c.Check(latest, equals, "") - t.CheckOutputLines( - "ERROR: Cannot find latest version of \"^python[0-9]+$\" in \"~/lang\".") -} - -func (s *Suite) Test_Pkgsrc_Latest_no_subdirs(c *check.C) { - t := s.Init(c) - - t.SetupFileLines("lang/Makefile") - - latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") - - c.Check(latest, equals, "") - t.CheckOutputLines( - "ERROR: Cannot find latest version of \"^python[0-9]+$\" in \"~/lang\".") -} - -func (s *Suite) Test_Pkgsrc_Latest_single(c *check.C) { - t := s.Init(c) - - t.SetupFileLines("lang/Makefile") - t.SetupFileLines("lang/python27/Makefile") - - latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") - - c.Check(latest, equals, "../../lang/python27") -} - -func (s *Suite) Test_Pkgsrc_Latest_multi(c *check.C) { - t := s.Init(c) - - t.SetupFileLines("lang/Makefile") - t.SetupFileLines("lang/python27/Makefile") - t.SetupFileLines("lang/python35/Makefile") - - latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") - - c.Check(latest, equals, "../../lang/python35") -} - -func (s *Suite) Test_Pkgsrc_Latest_numeric(c *check.C) { - t := s.Init(c) - - t.SetupFileLines("databases/postgresql95/Makefile") - t.SetupFileLines("databases/postgresql97/Makefile") - t.SetupFileLines("databases/postgresql100/Makefile") - t.SetupFileLines("databases/postgresql104/Makefile") - - latest := G.Pkgsrc.Latest("databases", `^postgresql[0-9]+$`, "$0") - - c.Check(latest, equals, "postgresql104") -} - // Demonstrates that an ALTERNATIVES file can be tested individually, // without any dependencies on a whole package or a PLIST file. func (s *Suite) Test_Pkglint_Checkfile__alternatives(c *check.C) { @@ -533,3 +478,300 @@ func (s *Suite) Test_Pkglint_Checkfile__ "WARN: log: Unexpected file found.", "0 errors and 1 warning found.") } + +func (s *Suite) Test_Pkglint_Tool__prefer_mk_over_pkgsrc(c *check.C) { + t := s.Init(c) + + G.Mk = t.NewMkLines("Makefile", MkRcsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", dummyMkLine) + local := G.Mk.Tools.Define("tool", "TOOL", dummyMkLine) + + global.Validity = Nowhere + local.Validity = AtRunTime + + loadTimeTool, loadTimeUsable := G.Tool("tool", LoadTime) + runTimeTool, runTimeUsable := G.Tool("tool", RunTime) + + c.Check(loadTimeTool, equals, local) + c.Check(loadTimeUsable, equals, false) + c.Check(runTimeTool, equals, local) + c.Check(runTimeUsable, equals, true) +} + +func (s *Suite) Test_Pkglint_Tool__lookup_by_name_fallback(c *check.C) { + t := s.Init(c) + + G.Mk = t.NewMkLines("Makefile", MkRcsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", dummyMkLine) + + global.Validity = Nowhere + + loadTimeTool, loadTimeUsable := G.Tool("tool", LoadTime) + runTimeTool, runTimeUsable := G.Tool("tool", RunTime) + + c.Check(loadTimeTool, equals, global) + c.Check(loadTimeUsable, equals, false) + c.Check(runTimeTool, equals, global) + c.Check(runTimeUsable, equals, false) +} + +func (s *Suite) Test_Pkglint_Tool__lookup_by_varname(c *check.C) { + t := s.Init(c) + + G.Mk = t.NewMkLines("Makefile", MkRcsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", dummyMkLine) + local := G.Mk.Tools.Define("tool", "TOOL", dummyMkLine) + + global.Validity = Nowhere + local.Validity = AtRunTime + + loadTimeTool, loadTimeUsable := G.Tool("${TOOL}", LoadTime) + runTimeTool, runTimeUsable := G.Tool("${TOOL}", RunTime) + + c.Check(loadTimeTool, equals, local) + c.Check(loadTimeUsable, equals, false) + c.Check(runTimeTool, equals, local) + c.Check(runTimeUsable, equals, true) +} + +func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback(c *check.C) { + t := s.Init(c) + + G.Mk = t.NewMkLines("Makefile", MkRcsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", dummyMkLine) + + global.Validity = Nowhere + + loadTimeTool, loadTimeUsable := G.Tool("${TOOL}", LoadTime) + runTimeTool, runTimeUsable := G.Tool("${TOOL}", RunTime) + + c.Check(loadTimeTool, equals, global) + c.Check(loadTimeUsable, equals, false) + c.Check(runTimeTool, equals, global) + c.Check(runTimeUsable, equals, false) +} + +func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback_runtime(c *check.C) { + t := s.Init(c) + + G.Mk = t.NewMkLines("Makefile", MkRcsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", dummyMkLine) + + global.Validity = AtRunTime + + loadTimeTool, loadTimeUsable := G.Tool("${TOOL}", LoadTime) + runTimeTool, runTimeUsable := G.Tool("${TOOL}", RunTime) + + c.Check(loadTimeTool, equals, global) + c.Check(loadTimeUsable, equals, false) + c.Check(runTimeTool, equals, global) + c.Check(runTimeUsable, equals, true) +} + +func (s *Suite) Test_Pkglint_ToolByVarname__prefer_mk_over_pkgsrc(c *check.C) { + t := s.Init(c) + + G.Mk = t.NewMkLines("Makefile", MkRcsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", dummyMkLine) + local := G.Mk.Tools.Define("tool", "TOOL", dummyMkLine) + + global.Validity = Nowhere + local.Validity = AtRunTime + + c.Check(G.ToolByVarname("TOOL", LoadTime), equals, local) + c.Check(G.ToolByVarname("TOOL", RunTime), equals, local) +} + +func (s *Suite) Test_Pkglint_ToolByVarname__fallback(c *check.C) { + t := s.Init(c) + + G.Mk = t.NewMkLines("Makefile", MkRcsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", dummyMkLine) + + global.Validity = AtRunTime + + c.Check(G.ToolByVarname("TOOL", LoadTime), equals, global) + c.Check(G.ToolByVarname("TOOL", RunTime), equals, global) +} + +func (s *Suite) Test_Pkglint_Checkfile__CheckExtra(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Call", "-Wall,no-space") + t.SetupPkgsrc() + G.Pkgsrc.LoadInfrastructure() + t.CreateFileLines("licenses/gnu-gpl-2.0") + t.CreateFileLines("category/Makefile") + t.CreateFileLines("category/package/Makefile", + MkRcsID, + "", + "DISTNAME= pkgname-1.0", + "CATEGORIES= category", + "", + "COMMENT= Comment", + "LICENSE= gnu-gpl-2.0", + "", + "NO_CHECKSUM= yes", + "", + ".include \"../../mk/bsd.pkg.mk\"") + t.CreateFileLines("category/package/PLIST", + PlistRcsID, + "bin/program") + t.CreateFileLines("category/package/INSTALL", + "#! /bin/sh") + t.CreateFileLines("category/package/DEINSTALL", + "#! /bin/sh") + + G.CheckDirent(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Pkglint_Checkfile__before_import(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Call", "-Wall,no-space", "--import") + t.SetupPkgsrc() + G.Pkgsrc.LoadInfrastructure() + t.CreateFileLines("licenses/gnu-gpl-2.0") + t.CreateFileLines("category/Makefile") + t.CreateFileLines("category/package/Makefile", + MkRcsID, + "", + "DISTNAME= pkgname-1.0", + "CATEGORIES= category", + "", + "COMMENT= Comment", + "LICENSE= gnu-gpl-2.0", + "", + "NO_CHECKSUM= yes", + "", + ".include \"../../mk/bsd.pkg.mk\"") + t.CreateFileLines("category/package/PLIST", + PlistRcsID, + "bin/program") + t.CreateFileLines("category/package/work/log") + t.CreateFileLines("category/package/Makefile~") + t.CreateFileLines("category/package/Makefile.orig") + t.CreateFileLines("category/package/Makefile.rej") + + G.CheckDirent(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/Makefile.orig: Must be cleaned up before committing the package.", + "ERROR: ~/category/package/Makefile.rej: Must be cleaned up before committing the package.", + "ERROR: ~/category/package/Makefile~: Must be cleaned up before committing the package.", + "ERROR: ~/category/package/work: Must be cleaned up before committing the package.") +} + +func (s *Suite) Test_Pkglint_Checkfile__errors(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Call", "-Wall,no-space") + t.SetupPkgsrc() + t.CreateFileLines("category/package/files/subdir/file") + t.CreateFileLines("category/package/files/subdir/subsub/file") + G.Pkgsrc.LoadInfrastructure() + + G.Checkfile(t.File("category/package/options.mk")) + G.Checkfile(t.File("category/package/files/subdir")) + G.Checkfile(t.File("category/package/files/subdir/subsub")) + G.Checkfile(t.File("category/package/files")) + + c.Check(t.Output(), check.Matches, `^`+ + `ERROR: ~/category/package/options.mk: Cannot determine file type: .*\n`+ + `WARN: ~/category/package/files/subdir/subsub: Unknown directory name\.\n`+ + `$`) +} + +func (s *Suite) Test_Pkglint_Checkfile__file_selection(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Call", "-Wall,no-space") + t.SetupPkgsrc() + t.CreateFileLines("doc/CHANGES-2018", + RcsID) + t.CreateFileLines("category/package/buildlink3.mk", + MkRcsID) + t.CreateFileLines("category/package/unexpected.txt", + RcsID) + G.Pkgsrc.LoadInfrastructure() + + G.Checkfile(t.File("doc/CHANGES-2018")) + G.Checkfile(t.File("category/package/buildlink3.mk")) + G.Checkfile(t.File("category/package/unexpected.txt")) + + t.CheckOutputLines( + "WARN: ~/category/package/buildlink3.mk:EOF: Expected a BUILDLINK_TREE line.", + "WARN: ~/category/package/unexpected.txt: Unexpected file found.") +} + +func (s *Suite) Test_Pkglint_Checkfile__readme_and_todo(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("category/Makefile", + MkRcsID) + + t.CreateFileLines("category/package/files/README", + "Extra file that is installed later.") + t.CreateFileLines("category/package/patches/patch-README", + RcsID, + "", + "Documentation", + "", + "--- old", + "+++ new", + "@@ -1,1 +1,1 @@", + "-old", + "+new") + t.CreateFileLines("category/package/Makefile", + MkRcsID, + "CATEGORIES=category", + "", + "COMMENT=Comment", + "LICENSE=2-clause-bsd") + t.CreateFileLines("category/package/PLIST", + PlistRcsID, + "bin/program") + t.CreateFileLines("category/package/README", + "This package ...") + t.CreateFileLines("category/package/TODO", + "Make this package work.") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "SHA1 (patch-README) = b9101ebf0bca8ce243ed6433b65555fa6a5ecd52") + + // Copy category/package to wip/package. + for _, basename := range []string{"files/README", "patches/patch-README", "Makefile", "PLIST", "README", "TODO", "distinfo"} { + src := "category/package/" + basename + dst := "wip/package/" + basename + text, err := ioutil.ReadFile(t.File(src)) + c.Check(err, check.IsNil) + t.CreateFileLines(dst, strings.TrimSpace(string(text))) + } + + t.SetupPkgsrc() + G.Pkgsrc.LoadInfrastructure() + t.Chdir(".") + + G.Main("pkglint", "category/package", "wip/package") + + t.CheckOutputLines( + "ERROR: category/package/README: Packages in main pkgsrc must not have a README file.", + "ERROR: category/package/TODO: Packages in main pkgsrc must not have a TODO file.", + "2 errors and 0 warnings found.") + + // FIXME: Do this resetting properly + G.errors = 0 + G.warnings = 0 + G.logged = make(map[string]bool) + G.Main("pkglint", "--import", "category/package", "wip/package") + + t.CheckOutputLines( + "ERROR: category/package/README: Packages in main pkgsrc must not have a README file.", + "ERROR: category/package/TODO: Packages in main pkgsrc must not have a TODO file.", + "ERROR: wip/package/README: Must be cleaned up before committing the package.", + "ERROR: wip/package/TODO: Must be cleaned up before committing the package.", + "4 errors and 0 warnings found.") +} Index: pkgsrc/pkgtools/pkglint/files/plist.go diff -u pkgsrc/pkgtools/pkglint/files/plist.go:1.27 pkgsrc/pkgtools/pkglint/files/plist.go:1.28 --- pkgsrc/pkgtools/pkglint/files/plist.go:1.27 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/plist.go Wed Sep 5 17:56:22 2018 @@ -45,16 +45,16 @@ type PlistChecker struct { } type PlistLine struct { - line Line + Line condition string // e.g. PLIST.docs - text string // Like line.text, without the condition + text string // Line.Text without any conditions of the form ${PLIST.cond} } func (ck *PlistChecker) Check(plainLines []Line) { plines := ck.NewLines(plainLines) ck.collectFilesAndDirs(plines) - if fname := plines[0].line.Filename; path.Base(fname) == "PLIST.common_end" { + if fname := plines[0].Filename; path.Base(fname) == "PLIST.common_end" { commonLines := Load(strings.TrimSuffix(fname, "_end"), NotEmpty) if commonLines != nil { ck.collectFilesAndDirs(ck.NewLines(commonLines)) @@ -123,22 +123,22 @@ func (ck *PlistChecker) checkline(pline text := pline.text if hasAlnumPrefix(text) { ck.checkpath(pline) - } else if m, cmd, arg := match2(text, `^(?:\$\{[\w.]+\})?@([a-z-]+)\s+(.*)`); m { + } else if m, cmd, arg := match2(text, `^(?:\$\{[\w.]+\})?@([a-z-]+)\s*(.*)`); m { pline.CheckDirective(cmd, arg) } else if hasPrefix(text, "$") { ck.checkpath(pline) } else if text == "" { - fix := pline.line.Autofix() + fix := pline.Autofix() fix.Warnf("PLISTs should not contain empty lines.") fix.Delete() fix.Apply() } else { - pline.line.Warnf("Unknown line type.") + pline.Warnf("Unknown line type: %s", pline.Line.Text) } } func (ck *PlistChecker) checkpath(pline *PlistLine) { - line, text := pline.line, pline.text + text := pline.text sdirname, basename := path.Split(text) dirname := strings.TrimSuffix(sdirname, "/") @@ -149,7 +149,7 @@ func (ck *PlistChecker) checkpath(pline pline.warnImakeMannewsuffix() } if hasPrefix(text, "${PKGMANDIR}/") { - fix := pline.line.Autofix() + fix := pline.Autofix() fix.Notef("PLIST files should mention \"man/\" instead of \"${PKGMANDIR}\".") fix.Explain( "The pkgsrc infrastructure takes care of replacing the correct value", @@ -169,7 +169,7 @@ func (ck *PlistChecker) checkpath(pline case "bin": ck.checkpathBin(pline, dirname, basename) case "doc": - line.Errorf("Documentation must be installed under share/doc, not doc.") + pline.Errorf("Documentation must be installed under share/doc, not doc.") case "etc": ck.checkpathEtc(pline, dirname, basename) case "info": @@ -183,23 +183,23 @@ func (ck *PlistChecker) checkpath(pline } if contains(text, "${PKGLOCALEDIR}") && G.Pkg != nil && !G.Pkg.vars.Defined("USE_PKGLOCALEDIR") { - line.Warnf("PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR was not found.") + pline.Warnf("PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR was not found.") } if contains(text, "/CVS/") { - line.Warnf("CVS files should not be in the PLIST.") + pline.Warnf("CVS files should not be in the PLIST.") } if hasSuffix(text, ".orig") { - line.Warnf(".orig files should not be in the PLIST.") + pline.Warnf(".orig files should not be in the PLIST.") } if hasSuffix(text, "/perllocal.pod") { - line.Warnf("perllocal.pod files should not be in the PLIST.") + pline.Warnf("perllocal.pod files should not be in the PLIST.") Explain( "This file is handled automatically by the INSTALL/DEINSTALL scripts,", "since its contents changes frequently.") } if contains(text, ".egg-info/") { - line.Warnf("Include \"../../lang/python/egg.mk\" instead of listing .egg-info files directly.") + pline.Warnf("Include \"../../lang/python/egg.mk\" instead of listing .egg-info files directly.") } } @@ -207,7 +207,7 @@ func (ck *PlistChecker) checkSorted(plin if text := pline.text; G.opts.WarnPlistSort && hasAlnumPrefix(text) && !containsVarRef(text) { if ck.lastFname != "" { if ck.lastFname > text && !G.opts.Autofix { - pline.line.Warnf("%q should be sorted before %q.", text, ck.lastFname) + pline.Warnf("%q should be sorted before %q.", text, ck.lastFname) Explain( "The files in the PLIST should be sorted alphabetically.", "To fix this, run \"pkglint -F PLIST\".") @@ -228,16 +228,16 @@ func (ck *PlistChecker) checkDuplicate(p return } - fix := pline.line.Autofix() + fix := pline.Autofix() fix.Errorf("Duplicate filename %q, already appeared in %s.", - text, prev.line.ReferenceFrom(pline.line)) + text, prev.ReferenceFrom(pline.Line)) fix.Delete() fix.Apply() } func (ck *PlistChecker) checkpathBin(pline *PlistLine, dirname, basename string) { if contains(dirname, "/") { - pline.line.Warnf("The bin/ directory should not have subdirectories.") + pline.Warnf("The bin/ directory should not have subdirectories.") Explain( "The programs in bin/ are collected there to be executable by the", "user without having to type an absolute path. This advantage does", @@ -249,22 +249,22 @@ func (ck *PlistChecker) checkpathBin(pli func (ck *PlistChecker) checkpathEtc(pline *PlistLine, dirname, basename string) { if hasPrefix(pline.text, "etc/rc.d/") { - pline.line.Errorf("RCD_SCRIPTS must not be registered in the PLIST. Please use the RCD_SCRIPTS framework.") + pline.Errorf("RCD_SCRIPTS must not be registered in the PLIST. Please use the RCD_SCRIPTS framework.") return } - pline.line.Errorf("Configuration files must not be registered in the PLIST. " + + pline.Errorf("Configuration files must not be registered in the PLIST. " + "Please use the CONF_FILES framework, which is described in mk/pkginstall/bsd.pkginstall.mk.") } func (ck *PlistChecker) checkpathInfo(pline *PlistLine, dirname, basename string) { if pline.text == "info/dir" { - pline.line.Errorf("\"info/dir\" must not be listed. Use install-info to add/remove an entry.") + pline.Errorf("\"info/dir\" must not be listed. Use install-info to add/remove an entry.") return } if G.Pkg != nil && !G.Pkg.vars.Defined("INFO_FILES") { - pline.line.Warnf("Packages that install info files should set INFO_FILES.") + pline.Warnf("Packages that install info files should set INFO_FILES in the Makefile.") } } @@ -274,62 +274,58 @@ func (ck *PlistChecker) checkpathLib(pli return case pline.text == "lib/charset.alias" && (G.Pkg == nil || G.Pkg.Pkgpath != "converters/libiconv"): - pline.line.Errorf("Only the libiconv package may install lib/charset.alias.") + pline.Errorf("Only the libiconv package may install lib/charset.alias.") return case hasPrefix(pline.text, "lib/locale/"): - pline.line.Errorf("\"lib/locale\" must not be listed. Use ${PKGLOCALEDIR}/locale and set USE_PKGLOCALEDIR instead.") + pline.Errorf("\"lib/locale\" must not be listed. Use ${PKGLOCALEDIR}/locale and set USE_PKGLOCALEDIR instead.") return } switch ext := path.Ext(basename); ext { - case ".a", ".la", ".so": - if ext == "la" { - if G.Pkg != nil && !G.Pkg.vars.Defined("USE_LIBTOOL") { - pline.line.Warnf("Packages that install libtool libraries should define USE_LIBTOOL.") - } + case ".la": + if G.Pkg != nil && !G.Pkg.vars.Defined("USE_LIBTOOL") { + pline.Warnf("Packages that install libtool libraries should define USE_LIBTOOL.") } } if contains(basename, ".a") || contains(basename, ".so") { if m, noext := match1(pline.text, `^(.*)(?:\.a|\.so[0-9.]*)$`); m { if laLine := ck.allFiles[noext+".la"]; laLine != nil { - pline.line.Warnf("Redundant library found. The libtool library is in %s.", - laLine.line.ReferenceFrom(pline.line)) + pline.Warnf("Redundant library found. The libtool library is in %s.", + laLine.ReferenceFrom(pline.Line)) } } } } func (ck *PlistChecker) checkpathMan(pline *PlistLine) { - line := pline.line - m, catOrMan, section, manpage, ext, gz := regex.Match5(pline.text, `^man/(cat|man)(\w+)/(.*?)\.(\w+)(\.gz)?$`) if !m { // maybe: line.Warnf("Invalid filename %q for manual page.", text) return } - if !matches(section, `^[\dln]$`) { - line.Warnf("Unknown section %q for manual page.", section) + if !matches(section, `^[0-9ln]$`) { + pline.Warnf("Unknown section %q for manual page.", section) } if catOrMan == "cat" && ck.allFiles["man/man"+section+"/"+manpage+"."+section] == nil { - line.Warnf("Preformatted manual page without unformatted one.") + pline.Warnf("Preformatted manual page without unformatted one.") } if catOrMan == "cat" { if ext != "0" { - line.Warnf("Preformatted manual pages should end in \".0\".") + pline.Warnf("Preformatted manual pages should end in \".0\".") } } else { if !hasPrefix(ext, section) { - line.Warnf("Mismatch between the section (%s) and extension (%s) of the manual page.", section, ext) + pline.Warnf("Mismatch between the section (%s) and extension (%s) of the manual page.", section, ext) } } if gz != "" { - fix := line.Autofix() + fix := pline.Autofix() fix.Notef("The .gz extension is unnecessary for manual pages.") fix.Explain( "Whether the manual pages are installed in compressed form or not is", @@ -342,20 +338,28 @@ func (ck *PlistChecker) checkpathMan(pli } func (ck *PlistChecker) checkpathShare(pline *PlistLine) { - line, text := pline.line, pline.text + text := pline.text switch { case hasPrefix(text, "share/icons/") && G.Pkg != nil: if hasPrefix(text, "share/icons/hicolor/") && G.Pkg.Pkgpath != "graphics/hicolor-icon-theme" { f := "../../graphics/hicolor-icon-theme/buildlink3.mk" if G.Pkg.included[f] == nil && ck.once.FirstTime("hicolor-icon-theme") { - line.Errorf("Packages that install hicolor icons must include %q in the Makefile.", f) + pline.Errorf("Packages that install hicolor icons must include %q in the Makefile.", f) } } + if text == "share/icons/hicolor/icon-theme.cache" && G.Pkg.Pkgpath != "graphics/hicolor-icon-theme" { + pline.Errorf("The file icon-theme.cache must not appear in any PLIST file.") + Explain( + "Remove this line and add the following line to the package Makefile.", + "", + ".include \"../../graphics/hicolor-icon-theme/buildlink3.mk\"") + } + if hasPrefix(text, "share/icons/gnome") && G.Pkg.Pkgpath != "graphics/gnome-icon-theme" { f := "../../graphics/gnome-icon-theme/buildlink3.mk" if G.Pkg.included[f] == nil { - line.Errorf("The package Makefile must include %q.", f) + pline.Errorf("The package Makefile must include %q.", f) Explain( "Packages that install GNOME icons must maintain the icon theme", "cache.") @@ -363,53 +367,47 @@ func (ck *PlistChecker) checkpathShare(p } if contains(text[12:], "/") && !G.Pkg.vars.Defined("ICON_THEMES") && ck.once.FirstTime("ICON_THEMES") { - line.Warnf("Packages that install icon theme files should set ICON_THEMES.") + pline.Warnf("Packages that install icon theme files should set ICON_THEMES.") } case hasPrefix(text, "share/doc/html/"): if G.opts.WarnPlistDepr { - line.Warnf("Use of \"share/doc/html\" is deprecated. Use \"share/doc/${PKGBASE}\" instead.") + pline.Warnf("Use of \"share/doc/html\" is deprecated. Use \"share/doc/${PKGBASE}\" instead.") } case G.Pkg != nil && G.Pkg.EffectivePkgbase != "" && (hasPrefix(text, "share/doc/"+G.Pkg.EffectivePkgbase+"/") || hasPrefix(text, "share/examples/"+G.Pkg.EffectivePkgbase+"/")): // Fine. - case text == "share/icons/hicolor/icon-theme.cache" && G.Pkg != nil && G.Pkg.Pkgpath != "graphics/hicolor-icon-theme": - line.Errorf("This file must not appear in any PLIST file.") - Explain( - "Remove this line and add the following line to the package Makefile.", - "", - ".include \"../../graphics/hicolor-icon-theme/buildlink3.mk\"") - case hasPrefix(text, "share/info/"): - line.Warnf("Info pages should be installed into info/, not share/info/.") + pline.Warnf("Info pages should be installed into info/, not share/info/.") Explain( - "To fix this, you should add INFO_FILES=yes to the package Makefile.") + "To fix this, add INFO_FILES=yes to the package Makefile.") case hasPrefix(text, "share/locale/") && hasSuffix(text, ".mo"): // Fine. case hasPrefix(text, "share/man/"): - line.Warnf("Man pages should be installed into man/, not share/man/.") + pline.Warnf("Man pages should be installed into man/, not share/man/.") } } func (pline *PlistLine) CheckTrailingWhitespace() { if hasSuffix(pline.text, " ") || hasSuffix(pline.text, "\t") { - pline.line.Errorf("pkgsrc does not support filenames ending in white-space.") + pline.Errorf("pkgsrc does not support filenames ending in white-space.") Explain( "Each character in the PLIST is relevant, even trailing white-space.") } } func (pline *PlistLine) CheckDirective(cmd, arg string) { - line := pline.line - if cmd == "unexec" { - if m, dir := match1(arg, `^(?:rmdir|\$\{RMDIR\} \%D/)(.*)`); m { + if m, dir := match1(arg, `^(?:rmdir|\$\{RMDIR\} %D/)(.*)`); m { if !contains(dir, "true") && !contains(dir, "${TRUE}") { - pline.line.Warnf("Please remove this line. It is no longer necessary.") + fix := pline.Autofix() + fix.Warnf("Please remove this line. It is no longer necessary.") + fix.Delete() + fix.Apply() } } } @@ -417,18 +415,15 @@ func (pline *PlistLine) CheckDirective(c switch cmd { case "exec", "unexec": switch { - case contains(arg, "install-info"), - contains(arg, "${INSTALL_INFO}"): - line.Warnf("@exec/unexec install-info is deprecated.") case contains(arg, "ldconfig") && !contains(arg, "/usr/bin/true"): - pline.line.Errorf("ldconfig must be used with \"||/usr/bin/true\".") + pline.Errorf("ldconfig must be used with \"||/usr/bin/true\".") } case "comment": // Nothing to do. case "dirrm": - line.Warnf("@dirrm is obsolete. Please remove this line.") + pline.Warnf("@dirrm is obsolete. Please remove this line.") Explain( "Directories are removed automatically when they are empty.", "When a package needs an empty directory, it can use the @pkgdir", @@ -438,7 +433,7 @@ func (pline *PlistLine) CheckDirective(c args := splitOnSpace(arg) switch { case len(args) != 3: - line.Warnf("Invalid number of arguments for imake-man.") + pline.Warnf("Invalid number of arguments for imake-man.") case args[2] == "${IMAKE_MANNEWSUFFIX}": pline.warnImakeMannewsuffix() } @@ -447,12 +442,12 @@ func (pline *PlistLine) CheckDirective(c // Nothing to check. default: - line.Warnf("Unknown PLIST directive \"@%s\".", cmd) + pline.Warnf("Unknown PLIST directive \"@%s\".", cmd) } } func (pline *PlistLine) warnImakeMannewsuffix() { - pline.line.Warnf("IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.") + pline.Warnf("IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.") Explain( "This is the result of a print-PLIST call that has not been edited", "manually by the package maintainer. Please replace the", @@ -492,7 +487,7 @@ func NewPlistLineSorter(plines []*PlistL for _, pline := range middle { if unsortable == nil && (hasPrefix(pline.text, "@") || contains(pline.text, "$")) { - unsortable = pline.line + unsortable = pline.Line } } return &plistLineSorter{header, middle, footer, unsortable, false, false} @@ -501,7 +496,7 @@ func NewPlistLineSorter(plines []*PlistL func (s *plistLineSorter) Sort() { if line := s.unsortable; line != nil { if G.opts.PrintAutofix || G.opts.Autofix { - line.Notef("This line prevents pkglint from sorting the PLIST automatically.") + trace.Stepf("%s: This line prevents pkglint from sorting the PLIST automatically.", line) } return } @@ -512,7 +507,7 @@ func (s *plistLineSorter) Sort() { if len(s.middle) == 0 { return } - firstLine := s.middle[0].line + firstLine := s.middle[0].Line sort.SliceStable(s.middle, func(i, j int) bool { mi := s.middle[i] @@ -536,13 +531,13 @@ func (s *plistLineSorter) Sort() { var lines []Line for _, pline := range s.header { - lines = append(lines, pline.line) + lines = append(lines, pline.Line) } for _, pline := range s.middle { - lines = append(lines, pline.line) + lines = append(lines, pline.Line) } for _, pline := range s.footer { - lines = append(lines, pline.line) + lines = append(lines, pline.Line) } s.autofixed = SaveAutofixChanges(lines) Index: pkgsrc/pkgtools/pkglint/files/util.go diff -u pkgsrc/pkgtools/pkglint/files/util.go:1.27 pkgsrc/pkgtools/pkglint/files/util.go:1.28 --- pkgsrc/pkgtools/pkglint/files/util.go:1.27 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/util.go Wed Sep 5 17:56:22 2018 @@ -139,25 +139,27 @@ func isCommitted(fname string) bool { func isLocallyModified(fname string) bool { lines := loadCvsEntries(fname) - needle := "/" + path.Base(fname) + "/" + if len(lines) == 0 { + return false + } + + baseName := path.Base(fname) for _, line := range lines { - if hasPrefix(line.Text, needle) { - cvsModTime, err := time.Parse(time.ANSIC, strings.Split(line.Text, "/")[3]) - if err != nil { - return false - } + fields := strings.Split(line.Text, "/") + if 3 < len(fields) && fields[1] == baseName { st, err := os.Stat(fname) if err != nil { - return false + return true } - // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724290(v=vs.85).aspx - // (System Services > Windows System Information > Time > About Time > File Times) - delta := cvsModTime.Unix() - st.ModTime().Unix() + // According to http://cvsman.com/cvs-1.12.12/cvs_19.php, format both timestamps. + cvsModTime := fields[3] + fsModTime := st.ModTime().Format(time.ANSIC) if trace.Tracing { - trace.Stepf("cvs.time=%v fs.time=%v delta=%v", cvsModTime, st.ModTime(), delta) + trace.Stepf("cvs.time=%q fs.time=%q", cvsModTime, st.ModTime()) } - return !(-2 <= delta && delta <= 2) + + return cvsModTime != fsModTime } } return false @@ -419,18 +421,18 @@ func NewScope() Scope { } // Define marks the variable and its canonicalized form as defined. -func (s *Scope) Define(varname string, line MkLine) { +func (s *Scope) Define(varname string, mkline MkLine) { if s.defined[varname] == nil { - s.defined[varname] = line + s.defined[varname] = mkline if trace.Tracing { - trace.Step2("Defining %q in line %s", varname, line.Linenos()) + trace.Step2("Defining %q in line %s", varname, mkline.Linenos()) } } varcanon := varnameCanon(varname) if varcanon != varname && s.defined[varcanon] == nil { - s.defined[varcanon] = line + s.defined[varcanon] = mkline if trace.Tracing { - trace.Step2("Defining %q in line %s", varcanon, line.Linenos()) + trace.Step2("Defining %q in line %s", varcanon, mkline.Linenos()) } } } @@ -454,6 +456,9 @@ func (s *Scope) Use(varname string, line // Defined tests whether the variable is defined. // It does NOT test the canonicalized variable name. +// +// Even if Defined returns true, FirstDefinition doesn't necessarily return true +// since the latter ignores the default definitions from vardefs.go, keyword dummyVardefMkline. func (s *Scope) Defined(varname string) bool { return s.defined[varname] != nil } @@ -490,14 +495,34 @@ func (s *Scope) UsedSimilar(varname stri return s.used[varnameCanon(varname)] != nil } +// FirstDefinition returns the line in which the variable has been defined first. +// Having multiple definitions is typical in the branches of "if" statements. func (s *Scope) FirstDefinition(varname string) MkLine { - return s.defined[varname] + mkline := s.defined[varname] + if mkline != nil && mkline.IsVarassign() { + return mkline + } + return nil // See NewPackage and G.Pkgsrc.UserDefinedVars } func (s *Scope) FirstUse(varname string) MkLine { return s.used[varname] } +func (s *Scope) Value(varname string) (value string, found bool) { + mkline := s.FirstDefinition(varname) + if mkline != nil { + return mkline.Value(), true + } + return "", false +} + +func (s *Scope) DefineAll(other Scope) { + for varname, mkline := range other.defined { + s.Define(varname, mkline) + } +} + // The MIT License (MIT) // // Copyright (c) 2015 Frits van Bommel @@ -611,12 +636,12 @@ func (s *RedundantScope) Handle(mkline M value := mkline.Value() valueNovar := mkline.WithoutMakeVariables(value) if op == opAssignEval && value == valueNovar { - op = opAssign // They are effectively the same in this case. + op = opAssign // The two operators are effectively the same in this case. } existing, found := s.vars[varname] if !found { if op == opAssignShell || op == opAssignEval { - s.vars[varname] = nil + s.vars[varname] = nil // Won't be checked further. } else { if op == opAssignAppend { value = " " + value @@ -639,9 +664,8 @@ func (s *RedundantScope) Handle(mkline M if s.OnIgnore != nil { s.OnIgnore(existing.mkline, mkline) } - case opAssignShell: - case opAssignEval: - s.vars[varname] = nil + case opAssignShell, opAssignEval: + s.vars[varname] = nil // Won't be checked further. } } @@ -655,6 +679,14 @@ func (s *RedundantScope) Handle(mkline M } } -func (s *RedundantScope) IsConditional(varname string) bool { - return s.vars[varname] != nil +func IsPrefs(fileName string) bool { + switch path.Base(fileName) { + case "bsd.prefs.mk", + "bsd.fast.prefs.mk", + "bsd.builtin.mk", // mk/buildlink3/bsd.builtin.mk + "pkgconfig-builtin.mk", + "bsd.options.mk": + return true + } + return false } Index: pkgsrc/pkgtools/pkglint/files/shell_test.go diff -u pkgsrc/pkgtools/pkglint/files/shell_test.go:1.29 pkgsrc/pkgtools/pkglint/files/shell_test.go:1.30 --- pkgsrc/pkgtools/pkglint/files/shell_test.go:1.29 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/shell_test.go Wed Sep 5 17:56:22 2018 @@ -50,6 +50,34 @@ func (s *Suite) Test_splitIntoShellToken c.Check(rest, equals, "") } +func (s *Suite) Test_splitIntoShellTokens__finished_dquot(c *check.C) { + text := "\"\"" + words, rest := splitIntoShellTokens(dummyLine, text) + + c.Check(words, deepEquals, []string{"\"\""}) + c.Check(rest, equals, "") +} + +func (s *Suite) Test_splitIntoShellTokens__unfinished_dquot(c *check.C) { + text := "\t\"" + words, rest := splitIntoShellTokens(dummyLine, text) + + c.Check(words, check.IsNil) + c.Check(rest, equals, "\"") +} + +func (s *Suite) Test_splitIntoShellTokens__unescaped_dollar_in_dquot(c *check.C) { + t := s.Init(c) + + text := "echo \"$$\"" + words, rest := splitIntoShellTokens(dummyLine, text) + + c.Check(words, deepEquals, []string{"echo", "\"$$\""}) + c.Check(rest, equals, "") + + t.CheckOutputEmpty() +} + func (s *Suite) Test_splitIntoShellTokens__varuse_with_embedded_space_and_other_vars(c *check.C) { varuseWord := "${GCONF_SCHEMAS:@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}" words, rest := splitIntoShellTokens(dummyLine, varuseWord) @@ -116,12 +144,9 @@ func (s *Suite) Test_ShellLine_CheckShel "\t"+shellCommand) shline := NewShellLine(G.Mk.mklines[0]) - G.Mk.ForEach( - func(mkline MkLine) bool { - shline.CheckShellCommandLine(shline.mkline.ShellCommand()) - return true - }, - func(mkline MkLine) {}) + G.Mk.ForEach(func(mkline MkLine) { + shline.CheckShellCommandLine(shline.mkline.ShellCommand()) + }) } checkShellCommandLine("@# Comment") @@ -135,7 +160,7 @@ func (s *Suite) Test_ShellLine_CheckShel "WARN: fname:1: Unknown shell command \"echo\".", "WARN: fname:1: Unknown shell command \"echo\".") - t.SetupTool(&Tool{Name: "echo", Predefined: true}) + t.SetupToolUsable("echo", "") t.SetupVartypes() checkShellCommandLine("echo ${PKGNAME:Q}") // vucQuotPlain @@ -173,8 +198,7 @@ func (s *Suite) Test_ShellLine_CheckShel checkShellCommandLine("echo \"$$\"") // As seen by make(1); the shell sees: echo "$" t.CheckOutputLines( - "WARN: fname:1: Pkglint parse error in ShTokenizer.ShAtom at \"$$\\\"\" (quoting=d).", - "WARN: fname:1: Pkglint ShellLine.CheckShellCommand: parse error at [\"]") + "WARN: fname:1: Unescaped $ or strange shell variable found.") checkShellCommandLine("echo \"\\n\"") @@ -268,13 +292,10 @@ func (s *Suite) Test_ShellLine_CheckShel G.Mk = t.NewMkLines("fname", "\t"+shellCommand) - G.Mk.ForEach( - func(mkline MkLine) bool { - shline := NewShellLine(mkline) - shline.CheckShellCommandLine(mkline.ShellCommand()) - return true - }, - func(mkline MkLine) {}) + G.Mk.ForEach(func(mkline MkLine) { + shline := NewShellLine(mkline) + shline.CheckShellCommandLine(mkline.ShellCommand()) + }) } checkShellCommandLine("${STRIP} executable") @@ -295,7 +316,7 @@ func (s *Suite) Test_ShellLine_CheckShel t.SetupCommandLine("-Wall") t.SetupVartypes() - t.SetupTool(&Tool{Name: "echo", Predefined: true}) + t.SetupToolUsable("echo", "") G.Mk = t.NewMkLines("Makefile", "\techo ${PKGNAME:Q}") shline := NewShellLine(G.Mk.mklines[0]) @@ -311,7 +332,7 @@ func (s *Suite) Test_ShellLine_CheckShel t.SetupCommandLine("-Wall", "--show-autofix") t.SetupVartypes() - t.SetupTool(&Tool{Name: "echo", Predefined: true}) + t.SetupToolUsable("echo", "") G.Mk = t.NewMkLines("Makefile", "\techo ${PKGNAME:Q}") shline := NewShellLine(G.Mk.mklines[0]) @@ -329,11 +350,11 @@ func (s *Suite) Test_ShellLine_CheckShel t.SetupCommandLine("-Wall") t.SetupVartypes() - t.SetupTool(&Tool{Name: "cat", Predefined: true}) - t.SetupTool(&Tool{Name: "echo", Predefined: true}) - t.SetupTool(&Tool{Name: "printf", Predefined: true}) - t.SetupTool(&Tool{Name: "sed", Predefined: true}) - t.SetupTool(&Tool{Name: "right-side", Predefined: true}) + t.SetupToolUsable("cat", "") + t.SetupToolUsable("echo", "") + t.SetupToolUsable("printf", "") + t.SetupToolUsable("sed", "") + t.SetupToolUsable("right-side", "") G.Mk = t.NewMkLines("Makefile", "\t echo | right-side", "\t sed s,s,s, | right-side", @@ -362,7 +383,7 @@ func (s *Suite) Test_ShellLine_CheckShel t.SetupCommandLine("-Wall", "--autofix") t.SetupVartypes() - t.SetupTool(&Tool{Name: "echo", Predefined: true}) + t.SetupToolUsable("echo", "") G.Mk = t.NewMkLines("Makefile", "\techo ${PKGNAME:Q}") shline := NewShellLine(G.Mk.mklines[0]) @@ -390,22 +411,12 @@ func (s *Suite) Test_ShellLine_CheckShel c.Check(tokens, deepEquals, []string{text}) c.Check(rest, equals, "") - G.Mk.ForEach( - func(mkline MkLine) bool { - shline.CheckWord(text, false) - return true - }, - func(mkline MkLine) {}) + G.Mk.ForEach(func(mkline MkLine) { shline.CheckWord(text, false, RunTime) }) t.CheckOutputLines( "WARN: fname:1: Unknown shell command \"echo\".") - G.Mk.ForEach( - func(mkline MkLine) bool { - shline.CheckShellCommandLine(text) - return true - }, - func(mkline MkLine) {}) + G.Mk.ForEach(func(mkline MkLine) { shline.CheckShellCommandLine(text) }) // No parse errors t.CheckOutputLines( @@ -416,11 +427,10 @@ func (s *Suite) Test_ShellLine_CheckShel t := s.Init(c) t.SetupVartypes() + t.SetupToolUsable("pax", "") G.Mk = t.NewMkLines("fname", "# dummy") shline := NewShellLine(G.Mk.mklines[0]) - t.SetupTool(&Tool{Name: "pax", Varname: "PAX"}) - G.Mk.tools["pax"] = true shline.CheckShellCommandLine("pax -rwpp -s /.*~$$//g . ${DESTDIR}${PREFIX}") @@ -436,7 +446,7 @@ func (s *Suite) Test_ShellLine_CheckWord checkWord := func(shellWord string, checkQuoting bool) { shline := t.NewShellLine("dummy.mk", 1, "\t echo "+shellWord) - shline.CheckWord(shellWord, checkQuoting) + shline.CheckWord(shellWord, checkQuoting, RunTime) } checkWord("${${list}}", false) @@ -449,6 +459,19 @@ func (s *Suite) Test_ShellLine_CheckWord t.CheckOutputEmpty() // No warning for variables that are partly indirect. + // The unquoted $@ takes a different code path in pkglint than the quoted $@. + checkWord("$@", false) + + t.CheckOutputLines( + "WARN: dummy.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") + + // When $@ appears as part of a shell token, it takes another code path in pkglint. + checkWord("-$@-", false) + + t.CheckOutputLines( + "WARN: dummy.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") + + // The unquoted $@ takes a different code path in pkglint than the quoted $@. checkWord("\"$@\"", false) t.CheckOutputLines( @@ -483,16 +506,28 @@ func (s *Suite) Test_ShellLine_CheckWord shline := t.NewShellLine("fname", 1, "# dummy") - shline.CheckWord("/.*~$$//g", false) // Typical argument to pax(1). + shline.CheckWord("/.*~$$//g", false, RunTime) // Typical argument to pax(1). t.CheckOutputEmpty() } +func (s *Suite) Test_ShellLine_CheckWord__dollar_subshell(c *check.C) { + t := s.Init(c) + + shline := t.NewShellLine("fname", 1, "\t$$(echo output)") + + shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) + + t.CheckOutputLines( + "WARN: fname:1: Invoking subshells via $(...) is not portable enough.") +} + func (s *Suite) Test_ShellLine_CheckShellCommandLine__echo(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") - t.SetupTool(&Tool{Name: "echo", Varname: "ECHO", MustUseVarForm: true, Predefined: true}) + echo := t.SetupToolUsable("echo", "ECHO") + echo.MustUseVarForm = true G.Mk = t.NewMkLines("fname", "# dummy") mkline := t.NewMkLine("fname", 3, "# dummy") @@ -513,7 +548,7 @@ func (s *Suite) Test_ShellLine_CheckShel text := "\tfor f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done" shline := t.NewShellLine("Makefile", 3, text) - shline.mkline.Tokenize(shline.mkline.ShellCommand()) + shline.mkline.Tokenize(shline.mkline.ShellCommand(), true) shline.CheckShellCommandLine(text) t.CheckOutputLines( @@ -652,16 +687,31 @@ func (s *Suite) Test_ShellLine_unescapeB t := s.Init(c) shline := t.NewShellLine("dummy.mk", 13, "# dummy") - // foobar="`echo \"foo bar\"`" - text := "foobar=\"`echo \\\"foo bar\\\"`\"" + // foobar="`echo \"foo bar\" "\ " "three"`" + text := "foobar=\"`echo \\\"foo bar\\\" \"\\ \" \"three\"`\"" repl := textproc.NewPrefixReplacer(text) repl.AdvanceStr("foobar=\"`") backtCommand, newQuoting := shline.unescapeBackticks(text, repl, shqDquotBackt) - c.Check(backtCommand, equals, "echo \"foo bar\"") + c.Check(backtCommand, equals, "echo \"foo bar\" \"\\ \" \"three\"") c.Check(newQuoting, equals, shqDquot) c.Check(repl.Rest(), equals, "\"") + + t.CheckOutputLines( + "WARN: dummy.mk:13: Backslashes should be doubled inside backticks.") +} + +func (s *Suite) Test_ShellLine_unescapeBackticks__dquotBacktDquot(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("dummy.mk", 13, "\t var=\"`\"\"`\"") + + MkLineChecker{mkline}.Check() + + t.CheckOutputLines( + "WARN: dummy.mk:13: Double quotes inside backticks inside double quotes are error prone.", + "WARN: dummy.mk:13: Double quotes inside backticks inside double quotes are error prone.") } func (s *Suite) Test_ShellLine__variable_outside_quotes(c *check.C) { @@ -710,3 +760,69 @@ func (s *Suite) Test_ShellLine_CheckShel "WARN: Makefile:3: The Solaris /bin/sh does not support negation of shell commands.", "WARN: Makefile:3: Found absolute pathname: /etc/passwd") } + +func (s *Suite) Test_SimpleCommandChecker_handleForbiddenCommand(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkRcsID, + "", + "\t${RUN} ktrace; mktexlsr; strace; texconfig; truss") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: Makefile:3: \"ktrace\" must not be used in Makefiles.", + "ERROR: Makefile:3: \"mktexlsr\" must not be used in Makefiles.", + "ERROR: Makefile:3: \"strace\" must not be used in Makefiles.", + "ERROR: Makefile:3: \"texconfig\" must not be used in Makefiles.", + "ERROR: Makefile:3: \"truss\" must not be used in Makefiles.") +} + +func (s *Suite) Test_SimpleCommandChecker_checkPaxPe(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkRcsID, + "", + "do-install:", + "\t${RUN} pax -pe ${WRKSRC} ${DESTDIR}${PREFIX}", + "\t${RUN} ${PAX} -pe ${WRKSRC} ${DESTDIR}${PREFIX}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:4: Please use the -pp option to pax(1) instead of -pe.", + "WARN: Makefile:5: Please use the -pp option to pax(1) instead of -pe.") +} + +func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkRcsID, + "", + "do-install:", + "\t${RUN} ${ECHO} -n 'Computing...'", + "\t${RUN} ${ECHO_N} 'Computing...'", + "\t${RUN} ${ECHO} 'Computing...'") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:4: Please use ${ECHO_N} instead of \"echo -n\".") +} + +func (s *Suite) Test_SimpleCommandChecker_checkConditionalCd(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkRcsID, + "pre-configure:", + "\t${RUN} while cd ..; do printf .; done") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") +} Index: pkgsrc/pkgtools/pkglint/files/shtypes_test.go diff -u pkgsrc/pkgtools/pkglint/files/shtypes_test.go:1.4 pkgsrc/pkgtools/pkglint/files/shtypes_test.go:1.5 --- pkgsrc/pkgtools/pkglint/files/shtypes_test.go:1.4 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/shtypes_test.go Wed Sep 5 17:56:22 2018 @@ -1,7 +1,7 @@ package main import ( - check "gopkg.in/check.v1" + "gopkg.in/check.v1" ) func NewShAtom(typ ShAtomType, text string, quoting ShQuoting) *ShAtom { Index: pkgsrc/pkgtools/pkglint/files/substcontext.go diff -u pkgsrc/pkgtools/pkglint/files/substcontext.go:1.12 pkgsrc/pkgtools/pkglint/files/substcontext.go:1.13 --- pkgsrc/pkgtools/pkglint/files/substcontext.go:1.12 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/substcontext.go Wed Sep 5 17:56:22 2018 @@ -44,34 +44,41 @@ func (st *SubstContextStats) Or(other Su } func (ctx *SubstContext) Varassign(mkline MkLine) { - if !G.opts.WarnExtra { - return - } if trace.Tracing { trace.Stepf("SubstContext.Varassign %#v %v#", ctx.curr, ctx.inAllBranches) } varname := mkline.Varname() + varcanon := mkline.Varcanon() + varparam := mkline.Varparam() op := mkline.Op() value := mkline.Value() - if varname == "SUBST_CLASSES" || hasPrefix(varname, "SUBST_CLASSES.") { + if varcanon == "SUBST_CLASSES" || varcanon == "SUBST_CLASSES.*" { classes := splitOnSpace(value) if len(classes) > 1 { mkline.Warnf("Please add only one class at a time to SUBST_CLASSES.") } if ctx.id != "" && ctx.id != classes[0] { - if ctx.IsComplete() { - ctx.Finish(mkline) - } else { - mkline.Warnf("SUBST_CLASSES should only appear once in a SUBST block.") + complete := ctx.IsComplete() + id := ctx.id + ctx.Finish(mkline) + if !complete { + mkline.Warnf("Subst block %q should be finished before adding the next class to SUBST_CLASSES.", id) } } ctx.id = classes[0] return } - m, varbase, varparam := match2(varname, `^(SUBST_(?:STAGE|MESSAGE|FILES|SED|VARS|FILTER_CMD))\.([\-\w_]+)$`) - if !m { + switch varcanon { + case "SUBST_STAGE.*": + case "SUBST_MESSAGE.*": + case "SUBST_FILES.*": + case "SUBST_SED.*": + case "SUBST_VARS.*": + case "SUBST_FILTER_CMD.*": + + default: if ctx.id != "" { mkline.Warnf("Foreign variable %q in SUBST block.", varname) } @@ -94,12 +101,12 @@ func (ctx *SubstContext) Varassign(mklin ctx.id = varparam } else { mkline.Warnf("Variable %q does not match SUBST class %q.", varname, ctx.id) + return } - return } - switch varbase { - case "SUBST_STAGE": + switch varcanon { + case "SUBST_STAGE.*": ctx.dupString(mkline, &ctx.stage, varname, value) if value == "pre-patch" || value == "post-patch" { fix := mkline.Autofix() @@ -116,26 +123,34 @@ func (ctx *SubstContext) Varassign(mklin fix.Replace("post-patch", "pre-configure") fix.Apply() } - case "SUBST_MESSAGE": + + if G.Pkg != nil && (value == "pre-configure" || value == "post-configure") { + if noConfigureLine := G.Pkg.vars.FirstDefinition("NO_CONFIGURE"); noConfigureLine != nil { + mkline.Warnf("SUBST_STAGE %s has no effect when NO_CONFIGURE is set (in %s).", + value, noConfigureLine.ReferenceFrom(mkline.Line)) + Explain( + "To fix this properly, remove the definition of NO_CONFIGURE.") + } + } + + case "SUBST_MESSAGE.*": ctx.dupString(mkline, &ctx.message, varname, value) - case "SUBST_FILES": + case "SUBST_FILES.*": ctx.dupBool(mkline, &ctx.curr.seenFiles, varname, op, value) - case "SUBST_SED": + case "SUBST_SED.*": ctx.dupBool(mkline, &ctx.curr.seenSed, varname, op, value) ctx.curr.seenTransform = true - case "SUBST_VARS": + case "SUBST_VARS.*": ctx.dupBool(mkline, &ctx.curr.seenVars, varname, op, value) ctx.curr.seenTransform = true - case "SUBST_FILTER_CMD": + case "SUBST_FILTER_CMD.*": ctx.dupString(mkline, &ctx.filterCmd, varname, value) ctx.curr.seenTransform = true - default: - mkline.Warnf("Foreign variable %q in SUBST block.", varname) } } func (ctx *SubstContext) Directive(mkline MkLine) { - if ctx.id == "" || !G.opts.WarnExtra { + if ctx.id == "" { return } @@ -171,19 +186,21 @@ func (ctx *SubstContext) IsComplete() bo } func (ctx *SubstContext) Finish(mkline MkLine) { - if ctx.id == "" || !G.opts.WarnExtra { + if ctx.id == "" { return } + + id := ctx.id if ctx.stage == "" { - mkline.Warnf("Incomplete SUBST block: %s missing.", ctx.varname("SUBST_STAGE")) + mkline.Warnf("Incomplete SUBST block: SUBST_STAGE.%s missing.", id) } if !ctx.curr.seenFiles { - mkline.Warnf("Incomplete SUBST block: %s missing.", ctx.varname("SUBST_FILES")) + mkline.Warnf("Incomplete SUBST block: SUBST_FILES.%s missing.", id) } if !ctx.curr.seenTransform { - mkline.Warnf("Incomplete SUBST block: %s, %s or %s missing.", - ctx.varname("SUBST_SED"), ctx.varname("SUBST_VARS"), ctx.varname("SUBST_FILTER_CMD")) + mkline.Warnf("Incomplete SUBST block: SUBST_SED.%[1]s, SUBST_VARS.%[1]s or SUBST_FILTER_CMD.%[1]s missing.", id) } + ctx.id = "" ctx.stage = "" ctx.message = "" @@ -191,14 +208,6 @@ func (ctx *SubstContext) Finish(mkline M ctx.filterCmd = "" } -func (ctx *SubstContext) varname(varbase string) string { - if ctx.id != "" { - return varbase + "." + ctx.id - } else { - return varbase - } -} - func (ctx *SubstContext) dupString(mkline MkLine, pstr *string, varname, value string) { if *pstr != "" { mkline.Warnf("Duplicate definition of %q.", varname) Index: pkgsrc/pkgtools/pkglint/files/util_test.go diff -u pkgsrc/pkgtools/pkglint/files/util_test.go:1.12 pkgsrc/pkgtools/pkglint/files/util_test.go:1.13 --- pkgsrc/pkgtools/pkglint/files/util_test.go:1.12 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/util_test.go Wed Sep 5 17:56:22 2018 @@ -4,7 +4,10 @@ import ( "gopkg.in/check.v1" "netbsd.org/pkglint/regex" "netbsd.org/pkglint/textproc" + "os" + "runtime" "testing" + "time" ) func (s *Suite) Test_YesNoUnknown_String(c *check.C) { @@ -16,6 +19,7 @@ func (s *Suite) Test_YesNoUnknown_String func (s *Suite) Test_MkopSubst__middle(c *check.C) { c.Check(mkopSubst("pkgname", false, "kgna", false, "ri", ""), equals, "prime") c.Check(mkopSubst("pkgname", false, "pkgname", false, "replacement", ""), equals, "replacement") + c.Check(mkopSubst("aaaaaaa", false, "a", false, "b", ""), equals, "baaaaaa") } func (s *Suite) Test_MkopSubst__left(c *check.C) { @@ -48,6 +52,19 @@ func (s *Suite) Test_replaceFirst(c *che c.Check(rest, equals, "X+c+d") } +func (s *Suite) Test_mustMatch(c *check.C) { + c.Check( + func() { mustMatch("aaa", `b`) }, + check.Panics, + "mustMatch \"aaa\" \"b\"") +} + +func (s *Suite) Test_shorten(c *check.C) { + c.Check(shorten("aaaaa", 3), equals, "aaa...") + c.Check(shorten("aaaaa", 5), equals, "aaaaa") + c.Check(shorten("aaa", 5), equals, "aaa") +} + func (s *Suite) Test_tabLength(c *check.C) { c.Check(tabWidth("12345"), equals, 5) c.Check(tabWidth("\t"), equals, 8) @@ -68,6 +85,22 @@ func (s *Suite) Test_cleanpath(c *check. c.Check(cleanpath("dir/"), equals, "dir") } +func (s *Suite) Test_relpath(c *check.C) { + if runtime.GOOS == "windows" { + c.Check(func() { relpath("c:/", "d:/") }, check.Panics, "relpath \"c:/\", \"d:/\"") + } +} + +func (s *Suite) Test_abspath(c *check.C) { + t := s.Init(c) + + if runtime.GOOS == "windows" { + t.ExpectFatal( + func() { abspath("file\u0000name") }, + "FATAL: file\x00name: Cannot determine absolute path.") + } +} + func (s *Suite) Test_isEmptyDir_and_getSubdirs(c *check.C) { t := s.Init(c) @@ -87,15 +120,24 @@ func (s *Suite) Test_isEmptyDir_and_getS if absent := t.File("nonexistent"); true { c.Check(isEmptyDir(absent), equals, true) // Counts as empty. - func() { - defer t.ExpectFatalError() - getSubdirs(absent) // Panics with a pkglintFatal. - }() // The last group from the error message is localized, therefore the matching. - c.Check(t.Output(), check.Matches, `FATAL: ~/nonexistent: Cannot be read: open ~/nonexistent: (.+)\n`) + t.ExpectFatalMatches( + func() { getSubdirs(absent) }, + `FATAL: ~/nonexistent: Cannot be read: open ~/nonexistent: (.+)\n`) } } +func (s *Suite) Test_isEmptyDir__empty_subdir(c *check.C) { + t := s.Init(c) + + t.SetupFileLines("CVS/Entries", + "dummy") + t.CreateFileLines("subdir/CVS/Entries", + "dummy") + + c.Check(isEmptyDir(t.File(".")), equals, true) +} + func (s *Suite) Test_PrefixReplacer_Since(c *check.C) { repl := textproc.NewPrefixReplacer("hello, world") mark := repl.Mark() @@ -181,3 +223,86 @@ func (s *Suite) Test_splitOnSpace(c *che c.Check(splitOnSpace(" "), check.IsNil) c.Check(splitOnSpace(""), check.IsNil) } + +func (s *Suite) Test_isLocallyModified(c *check.C) { + t := s.Init(c) + + unmodified := t.CreateFileLines("unmodified") + modTime := time.Unix(1136239445, 0) + + err := os.Chtimes(unmodified, modTime, modTime) + c.Check(err, check.IsNil) + + st, err := os.Lstat(unmodified) + c.Check(err, check.IsNil) + + // Make sure that the file system has second precision and accuracy. + c.Check(st.ModTime(), check.DeepEquals, modTime) + + modified := t.CreateFileLines("modified") + + t.CreateFileLines("CVS/Entries", + "/unmodified//"+modTime.Format(time.ANSIC)+"//", + "/modified//"+modTime.Format(time.ANSIC)+"//", + "/enoent//"+modTime.Format(time.ANSIC)+"//") + + c.Check(isLocallyModified(unmodified), equals, false) + c.Check(isLocallyModified(modified), equals, true) + c.Check(isLocallyModified(t.File("enoent")), equals, true) + c.Check(isLocallyModified(t.File("not_mentioned")), equals, false) +} + +func (s *Suite) Test_Scope_Defined(c *check.C) { + t := s.Init(c) + + scope := NewScope() + scope.Define("VAR.param", t.NewMkLine("file.mk", 1, "VAR.param=value")) + + c.Check(scope.Defined("VAR.param"), equals, true) + c.Check(scope.Defined("VAR.other"), equals, false) + c.Check(scope.Defined("VARIABLE.*"), equals, false) + + c.Check(scope.DefinedSimilar("VAR.param"), equals, true) + c.Check(scope.DefinedSimilar("VAR.other"), equals, true) + c.Check(scope.DefinedSimilar("VARIABLE.*"), equals, false) +} + +func (s *Suite) Test_Scope_Used(c *check.C) { + t := s.Init(c) + + scope := NewScope() + scope.Use("VAR.param", t.NewMkLine("file.mk", 1, "\techo ${VAR.param}")) + + c.Check(scope.Used("VAR.param"), equals, true) + c.Check(scope.Used("VAR.other"), equals, false) + c.Check(scope.Used("VARIABLE.*"), equals, false) + + c.Check(scope.UsedSimilar("VAR.param"), equals, true) + c.Check(scope.UsedSimilar("VAR.other"), equals, true) + c.Check(scope.UsedSimilar("VARIABLE.*"), equals, false) +} + +func (s *Suite) Test_Scope_DefineAll(c *check.C) { + t := s.Init(c) + + src := NewScope() + + dst := NewScope() + dst.DefineAll(src) + + c.Check(dst.defined, check.HasLen, 0) + c.Check(dst.used, check.HasLen, 0) + + src.Define("VAR", t.NewMkLine("file.mk", 1, "VAR=value")) + dst.DefineAll(src) + + c.Check(dst.Defined("VAR"), equals, true) +} + +func (s *Suite) Test_naturalLess(c *check.C) { + c.Check(naturalLess("0", "a"), equals, true) + c.Check(naturalLess("a", "0"), equals, false) + c.Check(naturalLess("000", "0000"), equals, true) + c.Check(naturalLess("0000", "000"), equals, false) + c.Check(naturalLess("000", "000"), equals, false) +} Index: pkgsrc/pkgtools/pkglint/files/vardefs.go diff -u pkgsrc/pkgtools/pkglint/files/vardefs.go:1.44 pkgsrc/pkgtools/pkglint/files/vardefs.go:1.45 --- pkgsrc/pkgtools/pkglint/files/vardefs.go:1.44 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/vardefs.go Wed Sep 5 17:56:22 2018 @@ -20,20 +20,20 @@ import ( // can be used in Makefiles without triggering warnings about typos. func (src *Pkgsrc) InitVartypes() { - acl := func(varname string, kindOfList KindOfList, checker *BasicType, aclentries string) { - m := mustMatch(varname, `^([A-Z_.][A-Z0-9_]*)(|\*|\.\*)$`) + acl := func(varname string, kindOfList KindOfList, checker *BasicType, aclEntries string) { + m := mustMatch(varname, `^([A-Z_.][A-Z0-9_]*|@)(|\*|\.\*)$`) varbase, varparam := m[1], m[2] - vtype := &Vartype{kindOfList, checker, parseACLEntries(varname, aclentries), false} + vartype := &Vartype{kindOfList, checker, parseACLEntries(varname, aclEntries), false} if src.vartypes == nil { src.vartypes = make(map[string]*Vartype) } if varparam == "" || varparam == "*" { - src.vartypes[varbase] = vtype + src.vartypes[varbase] = vartype } if varparam == "*" || varparam == ".*" { - src.vartypes[varbase+".*"] = vtype + src.vartypes[varbase+".*"] = vartype } } @@ -247,11 +247,16 @@ func (src *Pkgsrc) InitVartypes() { usr("BIN_INSTALL_FLAGS", lkShell, BtShellWord) usr("LOCALPATCHES", lkNone, BtPathname) - // The remaining variables from mk/defaults/mk.conf follow the - // naming conventions from MkLine.VariableType, furthermore - // they may be redefined by packages. Therefore they cannot be - // defined as user-defined. if false { + // The remaining variables from mk/defaults/mk.conf may be overridden by packages. + // Therefore they need a separate definition of "user-settable". + usr := func(varname string, kindOfList KindOfList, checker *BasicType) { + acl(varname, kindOfList, checker, ""+ + "Makefile: set, use; "+ + "buildlink3.mk, builtin.mk:; "+ + "Makefile.*, *.mk: default, set, use; "+ + "*: use-loadtime, use") + } usr("ACROREAD_FONTPATH", lkNone, BtPathlist) usr("AMANDA_USER", lkNone, BtUserGroupName) usr("AMANDA_TMP", lkNone, BtPathname) @@ -444,6 +449,7 @@ func (src *Pkgsrc) InitVartypes() { acl(".CURDIR", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") acl(".TARGET", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") + acl("@", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") acl("ALL_ENV", lkShell, BtShellWord, "") acl("ALTERNATIVES_FILE", lkNone, BtFilename, "") acl("ALTERNATIVES_SRC", lkShell, BtPathname, "") @@ -947,7 +953,7 @@ func (src *Pkgsrc) InitVartypes() { pkglist("PKG_SYSCONFDIR_PERMS", lkShell, BtPerms) sys("PKG_SYSCONFBASEDIR", lkNone, BtPathname) pkg("PKG_SYSCONFSUBDIR", lkNone, BtPathname) - acl("PKG_SYSCONFVAR", lkNone, BtIdentifier, "") // FIXME: name/type mismatch. + acl("PKG_SYSCONFVAR", lkNone, BtIdentifier, "") acl("PKG_UID", lkNone, BtInteger, "Makefile: set") acl("PKG_USERS", lkShell, BtShellWord, "Makefile: set, append") pkg("PKG_USERS_VARS", lkShell, BtVariableName) @@ -1107,42 +1113,25 @@ func (src *Pkgsrc) InitVartypes() { } func enum(values string) *BasicType { - vmap := make(map[string]bool) + valueMap := make(map[string]bool) for _, value := range splitOnSpace(values) { - vmap[value] = true + valueMap[value] = true } name := "enum: " + values + " " // See IsEnum - return &BasicType{name, func(cv *VartypeCheck) { - if cv.Op == opUseMatch { - if !vmap[cv.Value] && cv.Value == cv.ValueNoVar { - canMatch := false - for value := range vmap { - if ok, err := path.Match(cv.Value, value); err != nil { - cv.Line.Warnf("Invalid match pattern %q.", cv.Value) - } else if ok { - canMatch = true - } - } - if !canMatch { - cv.Line.Warnf("The pattern %q cannot match any of { %s } for %s.", cv.Value, values, cv.Varname) - } - } - return - } - - if cv.Value == cv.ValueNoVar && !vmap[cv.Value] { - cv.Line.Warnf("%q is not valid for %s. Use one of { %s } instead.", cv.Value, cv.Varname, values) - } - }} + basicType := &BasicType{name, nil} + basicType.checker = func(check *VartypeCheck) { + check.Enum(valueMap, basicType) + } + return basicType } -func parseACLEntries(varname string, aclentries string) []ACLEntry { - if aclentries == "" { +func parseACLEntries(varname string, aclEntries string) []ACLEntry { + if aclEntries == "" { return nil } var result []ACLEntry prevperms := "(first)" - for _, arg := range strings.Split(aclentries, "; ") { + for _, arg := range strings.Split(aclEntries, "; ") { var globs, perms string if fields := strings.SplitN(arg, ": ", 2); len(fields) == 2 { globs, perms = fields[0], fields[1] Index: pkgsrc/pkgtools/pkglint/files/vardefs_test.go diff -u pkgsrc/pkgtools/pkglint/files/vardefs_test.go:1.2 pkgsrc/pkgtools/pkglint/files/vardefs_test.go:1.3 --- pkgsrc/pkgtools/pkglint/files/vardefs_test.go:1.2 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/vardefs_test.go Wed Sep 5 17:56:22 2018 @@ -28,14 +28,11 @@ func (s *Suite) Test_InitVartypes__enumF "USE_LANGUAGES+= c++", ". endif", ".endfor") - mklines := t.SetupFileMkLines("Makefile", - MkRcsID, - "") t.SetupVartypes() checkEnumValues := func(varname, values string) { - vartype := mklines.mklines[1].VariableType(varname).String() + vartype := G.Pkgsrc.VariableType(varname).String() c.Check(vartype, equals, values) } Index: pkgsrc/pkgtools/pkglint/files/vartypecheck.go diff -u pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.38 pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.39 --- pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.38 Thu Aug 16 20:41:42 2018 +++ pkgsrc/pkgtools/pkglint/files/vartypecheck.go Wed Sep 5 17:56:22 2018 @@ -9,8 +9,10 @@ import ( ) type VartypeCheck struct { - MkLine MkLine - Line Line + MkLine MkLine + Line Line + + // The name of the variable being checked. In some cases it may also be the "description" of the variable. Varname string Op MkOperator Value string @@ -31,38 +33,6 @@ func NewVartypeCheckValue(vc *VartypeChe return © } -type MkOperator uint8 - -const ( - opAssign MkOperator = iota // = - opAssignShell // != - opAssignEval // := - opAssignAppend // += - opAssignDefault // ?= - opUseCompare // A variable is compared to a value, e.g. in a condition. - opUseMatch // A variable is matched using the :M or :N modifier. -) - -func NewMkOperator(op string) MkOperator { - switch op { - case "=": - return opAssign - case "!=": - return opAssignShell - case ":=": - return opAssignEval - case "+=": - return opAssignAppend - case "?=": - return opAssignDefault - } - return opAssign -} - -func (op MkOperator) String() string { - return [...]string{"=", "!=", ":=", "+=", "?=", "use", "use-loadtime", "use-match"}[op] -} - const ( reMachineOpsys = "" + // See mk/platform "AIX|BSDOS|Bitrig|Cygwin|Darwin|DragonFly|FreeBSD|FreeMiNT|GNUkFreeBSD|" + @@ -329,7 +299,7 @@ func (cv *VartypeCheck) DependencyWithPa MkLineChecker{cv.MkLine}.CheckRelativePkgdir(relpath) switch pkg { - case "msgfmt", "gettext": + case "gettext": line.Warnf("Please use USE_TOOLS+=msgfmt instead of this dependency.") case "perl5": line.Warnf("Please use USE_TOOLS+=perl:run instead of this dependency.") @@ -401,6 +371,30 @@ func (cv *VartypeCheck) EmulPlatform() { } } +func (cv *VartypeCheck) Enum(vmap map[string]bool, basicType *BasicType) { + if cv.Op == opUseMatch { + if !vmap[cv.Value] && cv.Value == cv.ValueNoVar { + canMatch := false + for value := range vmap { + if ok, err := path.Match(cv.Value, value); err != nil { + cv.Line.Warnf("Invalid match pattern %q.", cv.Value) + break + } else if ok { + canMatch = true + } + } + if !canMatch { + cv.Line.Warnf("The pattern %q cannot match any of { %s } for %s.", cv.Value, basicType.AllowedEnums(), cv.Varname) + } + } + return + } + + if cv.Value == cv.ValueNoVar && !vmap[cv.Value] { + cv.Line.Warnf("%q is not valid for %s. Use one of { %s } instead.", cv.Value, cv.Varname, basicType.AllowedEnums()) + } +} + func (cv *VartypeCheck) FetchURL() { MkLineChecker{cv.MkLine}.CheckVartypePrimitive(cv.Varname, BtURL, cv.Op, cv.Value, cv.MkComment, cv.Guessed) @@ -428,7 +422,8 @@ func (cv *VartypeCheck) FetchURL() { } } -// See Pathname +// See Pathname. +// // See http://www.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap03.html#tag_03_169 func (cv *VartypeCheck) Filename() { switch { @@ -442,10 +437,12 @@ func (cv *VartypeCheck) Filename() { } func (cv *VartypeCheck) Filemask() { - if cv.Op == opUseMatch { - return - } - if !matches(cv.ValueNoVar, `^[-0-9A-Za-z._~+%*?]*$`) { + switch { + case cv.Op == opUseMatch: + break + case contains(cv.ValueNoVar, "/"): + cv.Line.Warnf("A filename mask should not contain a slash.") + case !matches(cv.ValueNoVar, `^[#%*+\-./0-9?@A-Z\[\]_a-z~]*$`): cv.Line.Warnf("%q is not a valid filename mask.", cv.Value) } } @@ -454,7 +451,7 @@ func (cv *VartypeCheck) FileMode() { switch { case cv.Value != "" && cv.ValueNoVar == "": // Fine. - case matches(cv.Value, `^[0-7]{3,4}`): + case matches(cv.Value, `^[0-7]{3,4}$`): // Fine. default: cv.Line.Warnf("Invalid file mode %q.", cv.Value) @@ -467,9 +464,10 @@ func (cv *VartypeCheck) Homepage() { if m, wrong, sitename, subdir := match3(cv.Value, `^(\$\{(MASTER_SITE\w+)(?::=([\w\-/]+))?\})`); m { baseURL := G.Pkgsrc.MasterSiteVarToURL[sitename] if sitename == "MASTER_SITES" && G.Pkg != nil { - masterSites, _ := G.Pkg.varValue("MASTER_SITES") - if !containsVarRef(masterSites) { - baseURL = masterSites + if mkline := G.Pkg.vars.FirstDefinition("MASTER_SITES"); mkline != nil { + if masterSites := mkline.Value(); !containsVarRef(masterSites) { + baseURL = masterSites + } } } fixedURL := baseURL + subdir @@ -540,7 +538,7 @@ func (cv *VartypeCheck) LdFlag() { case hasPrefix(ldflag, "-"): cv.Line.Warnf("Unknown linker flag %q.", cv.Value) default: - cv.Line.Warnf("Linker flag %q should start with a hypen.", cv.Value) + cv.Line.Warnf("Linker flag %q should start with a hyphen.", cv.Value) } } @@ -700,7 +698,7 @@ func (cv *VartypeCheck) Pathmask() { if cv.Op == opUseMatch { return } - if !matches(cv.ValueNoVar, `^[#\-0-9A-Za-z._~+%*?/\[\]]*`) { + if !matches(cv.ValueNoVar, `^[#%*+\-./0-9?@A-Z\[\]_a-z~]*$`) { cv.Line.Warnf("%q is not a valid pathname mask.", cv.Value) } CheckLineAbsolutePathname(cv.Line, cv.Value) @@ -938,16 +936,16 @@ func (cv *VartypeCheck) ShellCommand() { return } setE := true - NewShellLine(cv.MkLine).CheckShellCommand(cv.Value, &setE) + NewShellLine(cv.MkLine).CheckShellCommand(cv.Value, &setE, RunTime) } // Zero or more shell commands, each terminated with a semicolon. func (cv *VartypeCheck) ShellCommands() { - NewShellLine(cv.MkLine).CheckShellCommands(cv.Value) + NewShellLine(cv.MkLine).CheckShellCommands(cv.Value, RunTime) } func (cv *VartypeCheck) ShellWord() { - NewShellLine(cv.MkLine).CheckWord(cv.Value, true) + NewShellLine(cv.MkLine).CheckWord(cv.Value, true, RunTime) } func (cv *VartypeCheck) Stage() { @@ -962,9 +960,10 @@ func (cv *VartypeCheck) Tool() { // no warning for package-defined tool definitions } else if m, toolname, tooldep := match2(cv.Value, `^([-\w]+|\[)(?::(\w+))?$`); m { - if G.Pkgsrc.Tools.ByName(toolname) == nil && (G.Mk == nil || G.Mk.toolRegistry.ByName(toolname) == nil) { + if tool, _ := G.Tool(toolname, RunTime); tool == nil { cv.Line.Errorf("Unknown tool %q.", toolname) } + switch tooldep { case "", "bootstrap", "build", "pkgsrc", "run", "test": default: @@ -992,7 +991,10 @@ func (cv *VartypeCheck) URL() { } else if m, _, host, _, _ := match4(value, `^(https?|ftp|gopher)://([-0-9A-Za-z.]+)(?::(\d+))?/([-%&+,./0-9:;=?@A-Z_a-z~]|#)*$`); m { if matches(host, `(?i)\.NetBSD\.org$`) && !matches(host, `\.NetBSD\.org$`) { - line.Warnf("Please write NetBSD.org instead of %s.", host) + fix := line.Autofix() + fix.Warnf("Please write NetBSD.org instead of %s.", host) + fix.ReplaceRegex(`(?i)NetBSD\.org`, "NetBSD.org", 1) + fix.Apply() } } else if m, scheme, _, absPath := match3(value, `^([0-9A-Za-z]+)://([^/]+)(.*)$`); m { Index: pkgsrc/pkgtools/pkglint/files/getopt/getopt_test.go diff -u pkgsrc/pkgtools/pkglint/files/getopt/getopt_test.go:1.4 pkgsrc/pkgtools/pkglint/files/getopt/getopt_test.go:1.5 --- pkgsrc/pkgtools/pkglint/files/getopt/getopt_test.go:1.4 Sat Jan 27 18:50:37 2018 +++ pkgsrc/pkgtools/pkglint/files/getopt/getopt_test.go Wed Sep 5 17:56:22 2018 @@ -1,7 +1,7 @@ package getopt import ( - check "gopkg.in/check.v1" + "gopkg.in/check.v1" "testing" ) Index: pkgsrc/pkgtools/pkglint/files/textproc/prefixreplacer.go diff -u pkgsrc/pkgtools/pkglint/files/textproc/prefixreplacer.go:1.4 pkgsrc/pkgtools/pkglint/files/textproc/prefixreplacer.go:1.5 --- pkgsrc/pkgtools/pkglint/files/textproc/prefixreplacer.go:1.4 Mon Feb 19 12:40:38 2018 +++ pkgsrc/pkgtools/pkglint/files/textproc/prefixreplacer.go Wed Sep 5 17:56:22 2018 @@ -55,6 +55,15 @@ func (pr *PrefixReplacer) AdvanceStr(pre return false } +func (pr *PrefixReplacer) AdvanceByte(b byte) bool { + if len(pr.rest) != 0 && pr.rest[0] == b { + pr.s = pr.rest[:1] + pr.rest = pr.rest[1:] + return true + } + return false +} + func (pr *PrefixReplacer) AdvanceBytesFunc(fn func(c byte) bool) bool { i := 0 for i < len(pr.rest) && fn(pr.rest[i]) { @@ -68,6 +77,7 @@ func (pr *PrefixReplacer) AdvanceBytesFu return false } +// AdvanceHspace advances over as many spaces and tabs as possible. func (pr *PrefixReplacer) AdvanceHspace() bool { i := 0 rest := pr.rest @@ -137,3 +147,11 @@ func (pr *PrefixReplacer) AdvanceRest() pr.rest = "" return rest } + +func (pr *PrefixReplacer) HasPrefix(str string) bool { + return strings.HasPrefix(pr.rest, str) +} + +func (pr *PrefixReplacer) HasPrefixRegexp(re regex.Pattern) bool { + return regex.Matches(pr.rest, re) +} Index: pkgsrc/pkgtools/pkglint/files/trace/tracing.go diff -u pkgsrc/pkgtools/pkglint/files/trace/tracing.go:1.1 pkgsrc/pkgtools/pkglint/files/trace/tracing.go:1.2 --- pkgsrc/pkgtools/pkglint/files/trace/tracing.go:1.1 Tue Jan 17 22:37:28 2017 +++ pkgsrc/pkgtools/pkglint/files/trace/tracing.go Wed Sep 5 17:56:22 2018 @@ -16,16 +16,6 @@ var ( var traceDepth int -func Ref(rv interface{}) ref { - return ref{rv} -} - -func (r ref) String() string { - ptr := reflect.ValueOf(r.intf) - ref := reflect.Indirect(ptr) - return fmt.Sprintf("%v", ref) -} - func Stepf(format string, args ...interface{}) { if Tracing { msg := fmt.Sprintf(format, args...) @@ -53,14 +43,17 @@ func Call2(arg1, arg2 string) func() { return traceCall(arg1, arg2) } +// Call records a function call in the tracing log, both when entering and +// when leaving the function. +// +// Usage: +// if trace.Tracing { +// defer trace.Call(arg1, arg2, trace.Result(result1), trace.Result(result2))() +// } func Call(args ...interface{}) func() { return traceCall(args...) } -type ref struct { - intf interface{} -} - // http://stackoverflow.com/questions/13476349/check-for-nil-and-nil-interface-in-go func isNil(a interface{}) bool { defer func() { @@ -69,19 +62,19 @@ func isNil(a interface{}) bool { return a == nil || reflect.ValueOf(a).IsNil() } -func argsStr(args ...interface{}) string { - argsStr := "" - for i, arg := range args { - if i != 0 { - argsStr += ", " +func argsStr(args []interface{}) string { + rv := "" + for _, arg := range args { + if rv != "" { + rv += ", " } if str, ok := arg.(fmt.Stringer); ok && !isNil(str) { - argsStr += str.String() + rv += str.String() } else { - argsStr += fmt.Sprintf("%#v", arg) + rv += fmt.Sprintf("%#v", arg) } } - return argsStr + return rv } func traceIndent() string { @@ -104,11 +97,49 @@ func traceCall(args ...interface{}) func } } indent := traceIndent() - io.WriteString(Out, fmt.Sprintf("TRACE: %s+ %s(%s)\n", indent, funcname, argsStr(args...))) + io.WriteString(Out, fmt.Sprintf("TRACE: %s+ %s(%s)\n", indent, funcname, argsStr(withoutResults(args)))) traceDepth++ return func() { traceDepth-- - io.WriteString(Out, fmt.Sprintf("TRACE: %s- %s(%s)\n", indent, funcname, argsStr(args...))) + io.WriteString(Out, fmt.Sprintf("TRACE: %s- %s(%s)\n", indent, funcname, argsStr(withResults(args)))) + } +} + +type result struct { + pointer interface{} +} + +// Result marks an argument as a result and is only logged when the function returns. +func Result(rv interface{}) result { + if reflect.ValueOf(rv).Kind() != reflect.Ptr { + panic(fmt.Sprintf("Result must be called with a pointer to the result, not %#v.", rv)) + } + return result{rv} +} + +func withoutResults(args []interface{}) []interface{} { + for i, arg := range args { + if _, ok := arg.(result); ok { + return args[0:i] + } + } + return args +} + +func withResults(args []interface{}) []interface{} { + for i, arg := range args { + if _, ok := arg.(result); ok { + var awr []interface{} + awr = append(awr, args[0:i]...) + awr = append(awr, "=>") + for _, res := range args[i:] { + pointer := reflect.ValueOf(res.(result).pointer) + actual := reflect.Indirect(pointer).Interface() + awr = append(awr, actual) + } + return awr + } } + return args } Added files: Index: pkgsrc/pkgtools/pkglint/files/trace/tracing_test.go diff -u /dev/null pkgsrc/pkgtools/pkglint/files/trace/tracing_test.go:1.1 --- /dev/null Wed Sep 5 17:56:23 2018 +++ pkgsrc/pkgtools/pkglint/files/trace/tracing_test.go Wed Sep 5 17:56:22 2018 @@ -0,0 +1,78 @@ +package trace + +import ( + "bytes" + "gopkg.in/check.v1" + "testing" +) + +type Suite struct{} + +var _ = check.Suite(new(Suite)) + +func Test(t *testing.T) { check.TestingT(t) } + +func onlyArguments(args ...interface{}) { + defer Call(args...)() + Stepf("Running %q", "code") +} + +func argumentsAndResult(arg0 string, arg1 int) (result string) { + defer Call(arg0, arg1, Result(&result))() + Stepf("Running %q", "code") + return "the result" +} + +func argumentsAndResultWrong(arg0 string, arg1 int) (result string) { + defer Call(arg0, arg1, result)() // "result" is evaluated too early and only once. + Stepf("Running %q", "code") + return "the result" +} + +func (s *Suite) Test_Call__onlyArguments(c *check.C) { + + output := s.captureTracingOutput(func() { + onlyArguments("arg0", 1234) + }) + + c.Check(output, check.Equals, ""+ + "TRACE: + netbsd.org/pkglint/trace.onlyArguments(\"arg0\", 1234)\n"+ + "TRACE: 1 Running \"code\"\n"+ + "TRACE: - netbsd.org/pkglint/trace.onlyArguments(\"arg0\", 1234)\n") +} + +func (s *Suite) Test_Call__argumentsAndResult(c *check.C) { + + output := s.captureTracingOutput(func() { + argumentsAndResult("arg0", 1234) + }) + + c.Check(output, check.Equals, ""+ + "TRACE: + netbsd.org/pkglint/trace.argumentsAndResult(\"arg0\", 1234)\n"+ + "TRACE: 1 Running \"code\"\n"+ + "TRACE: - netbsd.org/pkglint/trace.argumentsAndResult(\"arg0\", 1234, \"=>\", \"the result\")\n") +} + +func (s *Suite) Test_Call__argumentsAndResultWrong(c *check.C) { + + output := s.captureTracingOutput(func() { + argumentsAndResultWrong("arg0", 1234) + }) + + c.Check(output, check.Equals, ""+ + "TRACE: + netbsd.org/pkglint/trace.argumentsAndResultWrong(\"arg0\", 1234, \"\")\n"+ + "TRACE: 1 Running \"code\"\n"+ + "TRACE: - netbsd.org/pkglint/trace.argumentsAndResultWrong(\"arg0\", 1234, \"\")\n") +} + +func (s *Suite) captureTracingOutput(action func()) string { + out := bytes.Buffer{} + Out = &out + Tracing = true + + action() + + Tracing = false + Out = nil + return out.String() +} --_----------=_1536170183169240--