Received: by mail.netbsd.org (Postfix, from userid 605) id EFDD284F0F; Sat, 28 Apr 2018 23:32:55 +0000 (UTC) Received: from localhost (localhost [127.0.0.1]) by mail.netbsd.org (Postfix) with ESMTP id 44D5D84D78 for ; Sat, 28 Apr 2018 23:32:55 +0000 (UTC) X-Virus-Scanned: amavisd-new at netbsd.org Received: from mail.netbsd.org ([IPv6:::1]) by localhost (mail.netbsd.org [IPv6:::1]) (amavisd-new, port 10025) with ESMTP id c-zQBUYVcKIT for ; Sat, 28 Apr 2018 23:32:52 +0000 (UTC) Received: from cvs.NetBSD.org (ivanova.NetBSD.org [IPv6:2001:470:a085:999:28c:faff:fe03:5984]) by mail.netbsd.org (Postfix) with ESMTP id A76B784C78 for ; Sat, 28 Apr 2018 23:32:52 +0000 (UTC) Received: by cvs.NetBSD.org (Postfix, from userid 500) id 8CA07FBEC; Sat, 28 Apr 2018 23:32:52 +0000 (UTC) Content-Transfer-Encoding: 7bit Content-Type: multipart/mixed; boundary="_----------=_152495837221680" MIME-Version: 1.0 Date: Sat, 28 Apr 2018 23:32:52 +0000 From: "Roland Illig" Subject: CVS commit: pkgsrc/pkgtools/pkglint To: pkgsrc-changes@NetBSD.org Reply-To: rillig@netbsd.org X-Mailer: log_accum Message-Id: <20180428233252.8CA07FBEC@cvs.NetBSD.org> Sender: pkgsrc-changes-owner@NetBSD.org List-Id: pkgsrc-changes.NetBSD.org Precedence: bulk List-Unsubscribe: This is a multi-part message in MIME format. --_----------=_152495837221680 Content-Disposition: inline Content-Transfer-Encoding: 8bit Content-Type: text/plain; charset="US-ASCII" Module Name: pkgsrc Committed By: rillig Date: Sat Apr 28 23:32:52 UTC 2018 Modified Files: pkgsrc/pkgtools/pkglint: Makefile pkgsrc/pkgtools/pkglint/files: autofix.go licenses_test.go mkline_test.go mklines.go mktypes.go pkglint.go pkgsrc.go pkgsrc_test.go shell_test.go shtypes.go shtypes_test.go tools.go util.go util_test.go vardefs.go vartype.go vartypecheck.go vartypecheck_test.go Added Files: pkgsrc/pkgtools/pkglint/files: tools_test.go Log Message: Update pkglint to 5.5.9 Changes since 5.5.8: * Improved support for the "strip" tool, which has a special definition and is not directly connected to the STRIP variable. * Miscellaneous code cleanup and new tests. To generate a diff of this commit: cvs rdiff -u -r1.535 -r1.536 pkgsrc/pkgtools/pkglint/Makefile cvs rdiff -u -r1.6 -r1.7 pkgsrc/pkgtools/pkglint/files/autofix.go cvs rdiff -u -r1.11 -r1.12 pkgsrc/pkgtools/pkglint/files/licenses_test.go cvs rdiff -u -r1.32 -r1.33 pkgsrc/pkgtools/pkglint/files/mkline_test.go cvs rdiff -u -r1.22 -r1.23 pkgsrc/pkgtools/pkglint/files/mklines.go cvs rdiff -u -r1.4 -r1.5 pkgsrc/pkgtools/pkglint/files/mktypes.go cvs rdiff -u -r1.30 -r1.31 pkgsrc/pkgtools/pkglint/files/pkglint.go cvs rdiff -u -r1.1 -r1.2 pkgsrc/pkgtools/pkglint/files/pkgsrc.go \ pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go \ pkgsrc/pkgtools/pkglint/files/tools.go cvs rdiff -u -r1.23 -r1.24 pkgsrc/pkgtools/pkglint/files/shell_test.go \ pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go cvs rdiff -u -r1.7 -r1.8 pkgsrc/pkgtools/pkglint/files/shtypes.go cvs rdiff -u -r1.2 -r1.3 pkgsrc/pkgtools/pkglint/files/shtypes_test.go cvs rdiff -u -r0 -r1.1 pkgsrc/pkgtools/pkglint/files/tools_test.go cvs rdiff -u -r1.21 -r1.22 pkgsrc/pkgtools/pkglint/files/util.go cvs rdiff -u -r1.9 -r1.10 pkgsrc/pkgtools/pkglint/files/util_test.go cvs rdiff -u -r1.38 -r1.39 pkgsrc/pkgtools/pkglint/files/vardefs.go cvs rdiff -u -r1.13 -r1.14 pkgsrc/pkgtools/pkglint/files/vartype.go cvs rdiff -u -r1.31 -r1.32 pkgsrc/pkgtools/pkglint/files/vartypecheck.go Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. --_----------=_152495837221680 Content-Disposition: inline Content-Length: 26648 Content-Transfer-Encoding: binary Content-Type: text/x-diff; charset=us-ascii Modified files: Index: pkgsrc/pkgtools/pkglint/Makefile diff -u pkgsrc/pkgtools/pkglint/Makefile:1.535 pkgsrc/pkgtools/pkglint/Makefile:1.536 --- pkgsrc/pkgtools/pkglint/Makefile:1.535 Fri Apr 6 21:04:22 2018 +++ pkgsrc/pkgtools/pkglint/Makefile Sat Apr 28 23:32:52 2018 @@ -1,6 +1,6 @@ -# $NetBSD: Makefile,v 1.535 2018/04/06 21:04:22 rillig Exp $ +# $NetBSD: Makefile,v 1.536 2018/04/28 23:32:52 rillig Exp $ -PKGNAME= pkglint-5.5.8 +PKGNAME= pkglint-5.5.9 DISTFILES= # none CATEGORIES= pkgtools @@ -11,7 +11,6 @@ LICENSE= 2-clause-bsd CONFLICTS+= pkglint4-[0-9]* NO_CHECKSUM= yes -USE_LANGUAGES= c USE_TOOLS+= pax AUTO_MKDIRS= yes GO_SRCPATH= netbsd.org/pkglint Index: pkgsrc/pkgtools/pkglint/files/autofix.go diff -u pkgsrc/pkgtools/pkglint/files/autofix.go:1.6 pkgsrc/pkgtools/pkglint/files/autofix.go:1.7 --- pkgsrc/pkgtools/pkglint/files/autofix.go:1.6 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/autofix.go Sat Apr 28 23:32:52 2018 @@ -42,8 +42,10 @@ func NewAutofix(line Line) *Autofix { // because of the --only option. // // The fixer function must always call Describef. +// // If printAutofix or autofix is true, the fix should be done in // memory as far as possible (e.g. changes to the text of the line). +// // If autofix is true, the fix should be done permanently // (e.g. direct changes to the file system). func (fix *Autofix) Custom(fixer func(printAutofix, autofix bool)) { Index: pkgsrc/pkgtools/pkglint/files/licenses_test.go diff -u pkgsrc/pkgtools/pkglint/files/licenses_test.go:1.11 pkgsrc/pkgtools/pkglint/files/licenses_test.go:1.12 --- pkgsrc/pkgtools/pkglint/files/licenses_test.go:1.11 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/licenses_test.go Sat Apr 28 23:32:52 2018 @@ -1,7 +1,7 @@ package main import ( - check "gopkg.in/check.v1" + "gopkg.in/check.v1" ) func (s *Suite) Test_checklineLicense(c *check.C) { @@ -44,3 +44,50 @@ func (s *Suite) Test_checklineLicense(c t.CheckOutputEmpty() } + +func (s *Suite) Test_checkToplevelUnusedLicenses(c *check.C) { + t := s.Init(c) + + t.SetupFileLines("mk/bsd.pkg.mk", "# dummy") + t.SetupFileLines("mk/fetch/sites.mk", "# dummy") + t.SetupFileLines("mk/defaults/options.description", "option\tdescription") + t.SetupFileLines("doc/TODO") + t.SetupFileLines("mk/defaults/mk.conf") + t.SetupFileLines("mk/tools/bsd.tools.mk", + ".include \"actual-tools.mk\"") + t.SetupFileLines("mk/tools/actual-tools.mk") + t.SetupFileLines("mk/tools/defaults.mk") + t.SetupFileLines("mk/bsd.prefs.mk") + t.SetupFileLines("mk/misc/category.mk") + t.SetupFileLines("licenses/2-clause-bsd") + t.SetupFileLines("licenses/gnu-gpl-v3") + + t.SetupFileLines("Makefile", + MkRcsID, + "SUBDIR+=\tcategory") + + t.SetupFileLines("category/Makefile", + MkRcsID, + "COMMENT=\tExample category", + "", + "SUBDIR+=\tpackage", + "", + ".include \"../mk/misc/category.mk\"") + + t.SetupFileLines("category/package/Makefile", + MkRcsID, + "CATEGORIES=\tcategory", + "", + "COMMENT=Example package", + "LICENSE=\t2-clause-bsd", + "NO_CHECKSUM=\tyes") + t.SetupFileLines("category/package/PLIST", + PlistRcsID, + "bin/program") + + G.Main("pkglint", "-r", "-Cglobal", t.TmpDir()) + + t.CheckOutputLines( + "WARN: ~/licenses/gnu-gpl-v3: This license seems to be unused.", + "0 errors and 1 warning found.") +} Index: pkgsrc/pkgtools/pkglint/files/mkline_test.go diff -u pkgsrc/pkgtools/pkglint/files/mkline_test.go:1.32 pkgsrc/pkgtools/pkglint/files/mkline_test.go:1.33 --- pkgsrc/pkgtools/pkglint/files/mkline_test.go:1.32 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/mkline_test.go Sat Apr 28 23:32:52 2018 @@ -627,7 +627,7 @@ func (s *Suite) Test_MkLine_variableNeed t.SetupCommandLine("-Wall") t.SetupVartypes() - G.Pkgsrc.Tools.RegisterVarname("tar", "TAR") + G.Pkgsrc.Tools.RegisterVarname("tar", "TAR", dummyLine) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -648,7 +648,7 @@ func (s *Suite) Test_MkLine_variableNeed t.SetupCommandLine("-Wall") t.SetupVartypes() - G.Pkgsrc.Tools.RegisterVarname("cat", "CAT") + G.Pkgsrc.Tools.RegisterVarname("cat", "CAT", dummyLine) mklines := t.NewMkLines("Makefile", MkRcsID, "", Index: pkgsrc/pkgtools/pkglint/files/mklines.go diff -u pkgsrc/pkgtools/pkglint/files/mklines.go:1.22 pkgsrc/pkgtools/pkglint/files/mklines.go:1.23 --- pkgsrc/pkgtools/pkglint/files/mklines.go:1.22 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/mklines.go Sat Apr 28 23:32:52 2018 @@ -19,6 +19,7 @@ type MkLines struct { toolRegistry ToolRegistry // Tools defined in file scope. SeenBsdPrefsMk bool indentation Indentation // Indentation depth of preprocessing directives + // XXX: Why both tools and toolRegistry? } func NewMkLines(lines []Line) *MkLines { @@ -289,7 +290,7 @@ type VaralignBlock struct { type varalignBlockInfo struct { mkline MkLine varnameOp string // Variable name + assignment operator - varnameOpWidth int // Screen width of varnameOp + space + varnameOpWidth int // Screen width of varnameOp space string // Whitespace between varnameOp and the variable value totalWidth int // Screen width of varnameOp + space continuation bool // A continuation line with no value in the first line. Index: pkgsrc/pkgtools/pkglint/files/mktypes.go diff -u pkgsrc/pkgtools/pkglint/files/mktypes.go:1.4 pkgsrc/pkgtools/pkglint/files/mktypes.go:1.5 --- pkgsrc/pkgtools/pkglint/files/mktypes.go:1.4 Mon Feb 19 12:40:38 2018 +++ pkgsrc/pkgtools/pkglint/files/mktypes.go Sat Apr 28 23:32:52 2018 @@ -11,10 +11,15 @@ type MkToken struct { // MkVarUse represents a reference to a Make variable, with optional modifiers. // +// For nested variable expressions, the variable name can contain references +// to other variables. For example, ${TOOLS.${t}} is a MkVarUse with varname +// "TOOLS.${t}" and no modifiers. +// // Example: ${PKGNAME} +// // Example: ${PKGNAME:S/from/to/} type MkVarUse struct { - varname string + varname string // E.g. "PKGNAME", or "${BUILD_DEFS}" modifiers []string // E.g. "Q", "S/from/to/" } Index: pkgsrc/pkgtools/pkglint/files/pkglint.go diff -u pkgsrc/pkgtools/pkglint/files/pkglint.go:1.30 pkgsrc/pkgtools/pkglint/files/pkglint.go:1.31 --- pkgsrc/pkgtools/pkglint/files/pkglint.go:1.30 Fri Apr 6 21:04:22 2018 +++ pkgsrc/pkgtools/pkglint/files/pkglint.go Sat Apr 28 23:32:52 2018 @@ -119,8 +119,8 @@ func main() { } // Main runs the main program with the given arguments. -// args[0] is the program name. -func (pkglint *Pkglint) Main(args ...string) (exitcode int) { +// argv[0] is the program name. +func (pkglint *Pkglint) Main(argv ...string) (exitcode int) { defer func() { if r := recover(); r != nil { if _, ok := r.(pkglintFatal); ok { @@ -131,7 +131,7 @@ func (pkglint *Pkglint) Main(args ...str } }() - if exitcode := pkglint.ParseCommandLine(args); exitcode != nil { + if exitcode := pkglint.ParseCommandLine(argv); exitcode != nil { return *exitcode } Index: pkgsrc/pkgtools/pkglint/files/pkgsrc.go diff -u pkgsrc/pkgtools/pkglint/files/pkgsrc.go:1.1 pkgsrc/pkgtools/pkglint/files/pkgsrc.go:1.2 --- pkgsrc/pkgtools/pkglint/files/pkgsrc.go:1.1 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/pkgsrc.go Sat Apr 28 23:32:52 2018 @@ -14,7 +14,6 @@ import ( type Pkgsrc = *PkgsrcImpl type PkgsrcImpl struct { - // The top directory (PKGSRCDIR), either absolute or relative to // the current working directory. topdir string @@ -145,7 +144,8 @@ func (src *PkgsrcImpl) Latest(category s func (src *PkgsrcImpl) loadTools() { toolFiles := []string{"defaults.mk"} { - lines := G.Pkgsrc.LoadExistingLines("mk/tools/bsd.tools.mk", true) + toc := G.Pkgsrc.File("mk/tools/bsd.tools.mk") + lines := LoadExistingLines(toc, true) for _, line := range lines { if m, _, _, includefile := MatchMkInclude(line.Text); m { if !contains(includefile, "/") { @@ -154,16 +154,16 @@ func (src *PkgsrcImpl) loadTools() { } } if len(toolFiles) <= 1 { - lines[0].Fatalf("Too few tool files.") + NewLine(toc, 0, "", nil).Fatalf("Too few tool files.") } } reg := src.Tools - reg.RegisterTool(&Tool{"echo", "ECHO", true, true, true}) - reg.RegisterTool(&Tool{"echo -n", "ECHO_N", true, true, true}) - reg.RegisterTool(&Tool{"false", "FALSE", true /*why?*/, true, false}) - reg.RegisterTool(&Tool{"test", "TEST", true, true, true}) - reg.RegisterTool(&Tool{"true", "TRUE", true /*why?*/, true, true}) + reg.RegisterTool(&Tool{"echo", "ECHO", true, true, true}, dummyLine) + reg.RegisterTool(&Tool{"echo -n", "ECHO_N", true, true, true}, dummyLine) + reg.RegisterTool(&Tool{"false", "FALSE", true /*why?*/, true, false}, dummyLine) + reg.RegisterTool(&Tool{"test", "TEST", true, true, true}, dummyLine) + reg.RegisterTool(&Tool{"true", "TRUE", true /*why?*/, true, true}, dummyLine) for _, basename := range toolFiles { lines := G.Pkgsrc.LoadExistingLines("mk/tools/"+basename, true) @@ -190,11 +190,10 @@ func (src *PkgsrcImpl) loadTools() { if condDepth == 0 || condDepth == 1 && relativeName == "mk/bsd.prefs.mk" { for _, toolname := range splitOnSpace(value) { if !containsVarRef(toolname) { - for _, tool := range [...]*Tool{reg.Register(toolname), reg.Register("TOOLS_" + toolname)} { - tool.Predefined = true - if relativeName == "mk/bsd.prefs.mk" { - tool.UsableAtLoadtime = true - } + tool := reg.Register(toolname, line) + tool.Predefined = true + if relativeName == "mk/bsd.prefs.mk" { + tool.UsableAtLoadtime = true } } } @@ -357,7 +356,6 @@ func (src *PkgsrcImpl) loadUserDefinedVa func (src *PkgsrcImpl) initDeprecatedVars() { src.Deprecated = map[string]string{ - // December 2003 "FIX_RPATH": "It has been removed from pkgsrc in 2003.", @@ -519,7 +517,7 @@ func (src *PkgsrcImpl) initDeprecatedVar // LoadExistingLines loads the file relative to the pkgsrc top directory. func (src *PkgsrcImpl) LoadExistingLines(fileName string, joinBackslashLines bool) []Line { - return LoadExistingLines(src.topdir+"/"+fileName, joinBackslashLines) + return LoadExistingLines(src.File(fileName), joinBackslashLines) } // File resolves a file name relative to the pkgsrc top directory. Index: pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go diff -u pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go:1.1 pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go:1.2 --- pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go:1.1 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/pkgsrc_test.go Sat Apr 28 23:32:52 2018 @@ -66,7 +66,9 @@ func (s *Suite) Test_GlobalData_loadTool t.SetupFileLines("mk/tools/bsd.tools.mk", ".include \"flex.mk\"", - ".include \"gettext.mk\"") + ".include \"gettext.mk\"", + ".include \"strip.mk\"", + ".include \"replace.mk\"") t.SetupFileLines("mk/tools/defaults.mk", "_TOOLS_VARNAME.chown=CHOWN", "_TOOLS_VARNAME.gawk=AWK", @@ -77,8 +79,21 @@ func (s *Suite) Test_GlobalData_loadTool t.SetupFileLines("mk/tools/gettext.mk", "USE_TOOLS+=msgfmt", "TOOLS_CREATE+=msgfmt") + t.SetupFileLines("mk/tools/strip.mk", + ".if defined(_INSTALL_UNSTRIPPED) || !defined(TOOLS_PLATFORM.strip)", + "TOOLS_NOOP+= strip", + ".else", + "TOOLS_CREATE+= strip", + "TOOLS_PATH.strip= ${TOOLS_PLATFORM.strip}", + ".endif", + "STRIP?= strip") + t.SetupFileLines("mk/tools/replace.mk", + "_TOOLS.bzip2=\tbzip2 bzcat", + "#TOOLS_CREATE+=commented out", + "_UNRELATED_VAR=\t# empty") t.SetupFileLines("mk/bsd.prefs.mk", - "USE_TOOLS+=\tpwd") + "USE_TOOLS+=\tpwd", + "USE_TOOLS+=\tm4:pkgsrc") t.SetupFileLines("mk/bsd.pkg.mk", "USE_TOOLS+=\tmv") G.CurrentDir = t.TmpDir() @@ -91,16 +106,18 @@ func (s *Suite) Test_GlobalData_loadTool t.CheckOutputLines( "TRACE: + (*ToolRegistry).Trace()", - "TRACE: 1 tool &{Name:TOOLS_mv Varname: MustUseVarForm:false Predefined:true UsableAtLoadtime:false}", - "TRACE: 1 tool &{Name:TOOLS_pwd Varname: MustUseVarForm:false Predefined:true UsableAtLoadtime:true}", + "TRACE: 1 tool &{Name:bzcat Varname: MustUseVarForm:false Predefined:false UsableAtLoadtime:false}", + "TRACE: 1 tool &{Name:bzip2 Varname: MustUseVarForm:false Predefined:false UsableAtLoadtime:false}", "TRACE: 1 tool &{Name:chown Varname:CHOWN MustUseVarForm:false Predefined:false UsableAtLoadtime:false}", "TRACE: 1 tool &{Name:echo Varname:ECHO MustUseVarForm:true Predefined:true UsableAtLoadtime:true}", "TRACE: 1 tool &{Name:echo -n Varname:ECHO_N MustUseVarForm:true Predefined:true UsableAtLoadtime:true}", "TRACE: 1 tool &{Name:false Varname:FALSE MustUseVarForm:true Predefined:true UsableAtLoadtime:false}", "TRACE: 1 tool &{Name:gawk Varname:AWK MustUseVarForm:false Predefined:false UsableAtLoadtime:false}", + "TRACE: 1 tool &{Name:m4 Varname: MustUseVarForm:false Predefined:true UsableAtLoadtime:true}", "TRACE: 1 tool &{Name:msgfmt Varname: MustUseVarForm:false Predefined:false UsableAtLoadtime:false}", "TRACE: 1 tool &{Name:mv Varname:MV MustUseVarForm:false Predefined:true UsableAtLoadtime:false}", "TRACE: 1 tool &{Name:pwd Varname:PWD MustUseVarForm:false Predefined:true UsableAtLoadtime:true}", + "TRACE: 1 tool &{Name:strip Varname: MustUseVarForm:false Predefined:false UsableAtLoadtime:false}", "TRACE: 1 tool &{Name:test Varname:TEST MustUseVarForm:true Predefined:true UsableAtLoadtime:true}", "TRACE: 1 tool &{Name:true Varname:TRUE MustUseVarForm:true Predefined:true UsableAtLoadtime:true}", "TRACE: - (*ToolRegistry).Trace()") Index: pkgsrc/pkgtools/pkglint/files/tools.go diff -u pkgsrc/pkgtools/pkglint/files/tools.go:1.1 pkgsrc/pkgtools/pkglint/files/tools.go:1.2 --- pkgsrc/pkgtools/pkglint/files/tools.go:1.1 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/tools.go Sat Apr 28 23:32:52 2018 @@ -3,6 +3,7 @@ package main import ( "netbsd.org/pkglint/trace" "sort" + "strings" ) // See `mk/tools/`. @@ -23,23 +24,32 @@ func NewToolRegistry() ToolRegistry { return ToolRegistry{make(map[string]*Tool), make(map[string]*Tool)} } -func (tr *ToolRegistry) Register(toolname string) *Tool { - tool := tr.byName[toolname] +// Register registers the tool by its name. +// The tool may then be used by this name (e.g. "awk"), +// but not by a corresponding variable (e.g. ${AWK}). +// The toolname may include the scope (:pkgsrc, :run, etc.). +func (tr *ToolRegistry) Register(toolname string, line Line) *Tool { + name := strings.SplitN(toolname, ":", 2)[0] + tr.validateToolName(name, line) + + tool := tr.byName[name] if tool == nil { - tool = &Tool{Name: toolname} - tr.byName[toolname] = tool + tool = &Tool{Name: name} + tr.byName[name] = tool } return tool } -func (tr *ToolRegistry) RegisterVarname(toolname, varname string) *Tool { - tool := tr.Register(toolname) +func (tr *ToolRegistry) RegisterVarname(toolname, varname string, line Line) *Tool { + tool := tr.Register(toolname, line) tool.Varname = varname tr.byVarname[varname] = tool return tool } -func (tr *ToolRegistry) RegisterTool(tool *Tool) { +func (tr *ToolRegistry) RegisterTool(tool *Tool, line Line) { + tr.validateToolName(tool.Name, line) + if tool.Name != "" && tr.byName[tool.Name] == nil { tr.byName[tool.Name] = tool } @@ -78,24 +88,26 @@ func (tr *ToolRegistry) Trace() { } } +// ParseToolLine parses a tool definition from the pkgsrc infrastructure, +// e.g. in mk/tools/replace.mk. func (tr *ToolRegistry) ParseToolLine(line Line) { if m, commented, varname, _, _, _, value, _, _ := MatchVarassign(line.Text); m { if commented { return } if varname == "TOOLS_CREATE" && (value == "[" || matches(value, `^?[-\w.]+$`)) { - tr.Register(value) + tr.Register(value, line) } else if m, toolname := match1(varname, `^_TOOLS_VARNAME\.([-\w.]+|\[)$`); m { - tr.RegisterVarname(toolname, value) + tr.RegisterVarname(toolname, value, line) } else if m, toolname := match1(varname, `^(?:TOOLS_PATH|_TOOLS_DEPMETHOD)\.([-\w.]+|\[)$`); m { - tr.Register(toolname) + tr.Register(toolname, line) - } else if m, toolname := match1(varname, `_TOOLS\.(.*)`); m { - tr.Register(toolname) + } else if m, toolname := match1(varname, `^_TOOLS\.(.*)`); m { + tr.Register(toolname, line) for _, tool := range splitOnSpace(value) { - tr.Register(tool) + tr.Register(tool, line) } } } @@ -114,3 +126,9 @@ func (tr *ToolRegistry) ForEach(action f action(tool) } } + +func (tr *ToolRegistry) validateToolName(toolName string, line Line) { + if toolName != "echo -n" && !matches(toolName, `^([-a-z0-9.]+|\[)$`) { + line.Errorf("Invalid tool name %q", toolName) + } +} Index: pkgsrc/pkgtools/pkglint/files/shell_test.go diff -u pkgsrc/pkgtools/pkglint/files/shell_test.go:1.23 pkgsrc/pkgtools/pkglint/files/shell_test.go:1.24 --- pkgsrc/pkgtools/pkglint/files/shell_test.go:1.23 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/shell_test.go Sat Apr 28 23:32:52 2018 @@ -254,6 +254,32 @@ func (s *Suite) Test_ShellLine_CheckShel t.CheckOutputEmpty() // No warning about missing error checking. } +func (s *Suite) Test_ShellLine_CheckShellCommandLine_strip(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + + checkShellCommandLine := func(shellCommand string) { + G.Mk = t.NewMkLines("fname", + "\t"+shellCommand) + shline := NewShellLine(G.Mk.mklines[0]) + + shline.CheckShellCommandLine(shline.mkline.Shellcmd()) + } + + checkShellCommandLine("${STRIP} executable") + + t.CheckOutputLines( + "WARN: fname:1: Unknown shell command \"${STRIP}\".", + "WARN: fname:1: STRIP is used but not defined. Spelling mistake?") + + t.SetupVartypes() + + checkShellCommandLine("${STRIP} executable") + + t.CheckOutputEmpty() +} + func (s *Suite) Test_ShellLine_CheckShellCommandLine__nofix(c *check.C) { t := s.Init(c) Index: pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go diff -u pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go:1.23 pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go:1.24 --- pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go:1.23 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/vartypecheck_test.go Sat Apr 28 23:32:52 2018 @@ -541,6 +541,30 @@ func (s *Suite) Test_VartypeCheck_Stage( "Use one of {pre,do,post}-{extract,patch,configure,build,test,install}.") } +func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { + t := s.Init(c) + + t.SetupTool(&Tool{Name: "tool1", Predefined: true}) + t.SetupTool(&Tool{Name: "tool2", Predefined: true}) + t.SetupTool(&Tool{Name: "tool3", Predefined: true}) + + runVartypeChecks(t, "USE_TOOLS", opAssignAppend, (*VartypeCheck).Tool, + "tool3:run", + "tool2:unknown") + + t.CheckOutputLines( + "ERROR: fname:2: Unknown tool dependency \"unknown\". " + + "Use one of \"bootstrap\", \"build\", \"pkgsrc\" or \"run\".") + + runVartypeChecks(t, "USE_TOOLS.NetBSD", opAssignAppend, (*VartypeCheck).Tool, + "tool3:run", + "tool2:unknown") + + t.CheckOutputLines( + "ERROR: fname:2: Unknown tool dependency \"unknown\". " + + "Use one of \"bootstrap\", \"build\", \"pkgsrc\" or \"run\".") +} + func (s *Suite) Test_VartypeCheck_VariableName(c *check.C) { t := s.Init(c) Index: pkgsrc/pkgtools/pkglint/files/shtypes.go diff -u pkgsrc/pkgtools/pkglint/files/shtypes.go:1.7 pkgsrc/pkgtools/pkglint/files/shtypes.go:1.8 --- pkgsrc/pkgtools/pkglint/files/shtypes.go:1.7 Mon Feb 19 12:40:38 2018 +++ pkgsrc/pkgtools/pkglint/files/shtypes.go Sat Apr 28 23:32:52 2018 @@ -36,7 +36,6 @@ func (t ShAtomType) IsWord() bool { return false } -// @Beta type ShAtom struct { Type ShAtomType MkText string @@ -44,21 +43,21 @@ type ShAtom struct { Data interface{} } -func (token *ShAtom) String() string { - if token.Type == shtWord && token.Quoting == shqPlain && token.Data == nil { - return fmt.Sprintf("%q", token.MkText) +func (atom *ShAtom) String() string { + if atom.Type == shtWord && atom.Quoting == shqPlain && atom.Data == nil { + return fmt.Sprintf("%q", atom.MkText) } - if token.Type == shtVaruse { - varuse := token.Data.(*MkVarUse) + if atom.Type == shtVaruse { + varuse := atom.Data.(*MkVarUse) return fmt.Sprintf("varuse(%q)", varuse.varname+varuse.Mod()) } - return fmt.Sprintf("ShAtom(%v, %q, %s)", token.Type, token.MkText, token.Quoting) + return fmt.Sprintf("ShAtom(%v, %q, %s)", atom.Type, atom.MkText, atom.Quoting) } -// Returns nil for plain shell tokens. -func (token *ShAtom) VarUse() *MkVarUse { - if token.Type == shtVaruse { - return token.Data.(*MkVarUse) +// VarUse returns a read access to a Makefile variable, or nil for plain shell tokens. +func (atom *ShAtom) VarUse() *MkVarUse { + if atom.Type == shtVaruse { + return atom.Data.(*MkVarUse) } return nil } Index: pkgsrc/pkgtools/pkglint/files/shtypes_test.go diff -u pkgsrc/pkgtools/pkglint/files/shtypes_test.go:1.2 pkgsrc/pkgtools/pkglint/files/shtypes_test.go:1.3 --- pkgsrc/pkgtools/pkglint/files/shtypes_test.go:1.2 Sun Jul 10 21:24:47 2016 +++ pkgsrc/pkgtools/pkglint/files/shtypes_test.go Sat Apr 28 23:32:52 2018 @@ -12,10 +12,23 @@ func NewShAtomVaruse(text string, quotin return &ShAtom{shtVaruse, text, quoting, NewMkVarUse(varname, modifiers...)} } -func (s *Suite) Test_ShAtom_String(c *check.C) { +func (s *Suite) Test_ShAtomType_String(c *check.C) { c.Check(shtComment.String(), equals, "comment") } +func (s *Suite) Test_ShAtom_String(c *check.C) { + tokenizer := NewShTokenizer(dummyLine, "${ECHO} \"hello, world\"", false) + + atoms := tokenizer.ShAtoms() + + c.Check(len(atoms), equals, 5) + c.Check(atoms[0].String(), equals, "varuse(\"ECHO\")") + c.Check(atoms[1].String(), equals, "ShAtom(space, \" \", plain)") + c.Check(atoms[2].String(), equals, "ShAtom(word, \"\\\"\", d)") + c.Check(atoms[3].String(), equals, "ShAtom(word, \"hello, world\", d)") + c.Check(atoms[4].String(), equals, "\"\\\"\"") +} + func (s *Suite) Test_ShQuoting_String(c *check.C) { c.Check(shqDquotBacktSquot.String(), equals, "dbs") } Index: pkgsrc/pkgtools/pkglint/files/util.go diff -u pkgsrc/pkgtools/pkglint/files/util.go:1.21 pkgsrc/pkgtools/pkglint/files/util.go:1.22 --- pkgsrc/pkgtools/pkglint/files/util.go:1.21 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/util.go Sat Apr 28 23:32:52 2018 @@ -243,7 +243,7 @@ func varIsUsed(varname string) bool { } func splitOnSpace(s string) []string { - return regex.Compile(`\s+`).Split(s, -1) + return regex.Compile(`\S+`).FindAllString(s, -1) } func fileExists(fname string) bool { Index: pkgsrc/pkgtools/pkglint/files/util_test.go diff -u pkgsrc/pkgtools/pkglint/files/util_test.go:1.9 pkgsrc/pkgtools/pkglint/files/util_test.go:1.10 --- pkgsrc/pkgtools/pkglint/files/util_test.go:1.9 Sat Jan 27 18:50:36 2018 +++ pkgsrc/pkgtools/pkglint/files/util_test.go Sat Apr 28 23:32:52 2018 @@ -164,3 +164,10 @@ func emptyToNil(slice []string) []string } return slice } + +func (s *Suite) Test_splitOnSpace(c *check.C) { + c.Check(splitOnSpace("aaaaa"), deepEquals, []string{"aaaaa"}) + c.Check(splitOnSpace(" a b\tc\n"), deepEquals, []string{"a", "b", "c"}) + c.Check(splitOnSpace(" "), check.IsNil) + c.Check(splitOnSpace(""), check.IsNil) +} Index: pkgsrc/pkgtools/pkglint/files/vardefs.go diff -u pkgsrc/pkgtools/pkglint/files/vardefs.go:1.38 pkgsrc/pkgtools/pkglint/files/vardefs.go:1.39 --- pkgsrc/pkgtools/pkglint/files/vardefs.go:1.38 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/vardefs.go Sat Apr 28 23:32:52 2018 @@ -938,6 +938,7 @@ func (src *PkgsrcImpl) InitVartypes() { pkg("SMF_METHOD_SHELL", lkNone, BtShellCommand) pkglist("SPECIAL_PERMS", lkShell, BtPerms) sys("STEP_MSG", lkNone, BtShellCommand) + sys("STRIP", lkNone, BtShellCommand) // see mk/tools/strip.mk acl("SUBDIR", lkShell, BtFilename, "Makefile: append; *:") acl("SUBST_CLASSES", lkShell, BtIdentifier, "Makefile: set, append; *: append") acl("SUBST_CLASSES.*", lkShell, BtIdentifier, "Makefile: set, append; *: append") Index: pkgsrc/pkgtools/pkglint/files/vartype.go diff -u pkgsrc/pkgtools/pkglint/files/vartype.go:1.13 pkgsrc/pkgtools/pkglint/files/vartype.go:1.14 --- pkgsrc/pkgtools/pkglint/files/vartype.go:1.13 Mon Feb 19 12:40:38 2018 +++ pkgsrc/pkgtools/pkglint/files/vartype.go Sat Apr 28 23:32:52 2018 @@ -173,6 +173,7 @@ func (vt *Vartype) IsBasicSafe() bool { BtRelativePkgDir, BtRelativePkgPath, BtStage, + BtTool, // Sometimes contains a colon, but that should be ok. BtUserGroupName, BtVersion, BtWrkdirSubdirectory, Index: pkgsrc/pkgtools/pkglint/files/vartypecheck.go diff -u pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.31 pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.32 --- pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.31 Sat Mar 24 14:32:49 2018 +++ pkgsrc/pkgtools/pkglint/files/vartypecheck.go Sat Apr 28 23:32:52 2018 @@ -940,18 +940,19 @@ func (cv *VartypeCheck) Stage() { } } +// Tool checks for tool names like "awk", "m4:pkgsrc", "digest:bootstrap". func (cv *VartypeCheck) Tool() { if cv.Varname == "TOOLS_NOOP" && cv.Op == opAssignAppend { // no warning for package-defined tool definitions } else if m, toolname, tooldep := match2(cv.Value, `^([-\w]+|\[)(?::(\w+))?$`); m { - if G.Pkgsrc.Tools.ByName(toolname) == nil && (G.Mk == nil || G.Mk.toolRegistry.byName[toolname] == nil) { + if G.Pkgsrc.Tools.ByName(toolname) == nil && (G.Mk == nil || G.Mk.toolRegistry.ByName(toolname) == nil) { cv.Line.Errorf("Unknown tool %q.", toolname) } switch tooldep { case "", "bootstrap", "build", "pkgsrc", "run": default: - cv.Line.Errorf("Unknown tool dependency %q. Use one of \"build\", \"pkgsrc\" or \"run\".", tooldep) + cv.Line.Errorf("Unknown tool dependency %q. Use one of \"bootstrap\", \"build\", \"pkgsrc\" or \"run\".", tooldep) } } else if cv.Op != opUseMatch { cv.Line.Errorf("Invalid tool syntax: %q.", cv.Value) Added files: Index: pkgsrc/pkgtools/pkglint/files/tools_test.go diff -u /dev/null pkgsrc/pkgtools/pkglint/files/tools_test.go:1.1 --- /dev/null Sat Apr 28 23:32:52 2018 +++ pkgsrc/pkgtools/pkglint/files/tools_test.go Sat Apr 28 23:32:52 2018 @@ -0,0 +1,20 @@ +package main + +import "gopkg.in/check.v1" + +func (s *Suite) Test_ToolRegistry_ParseToolLine(c *check.C) { + t := s.Init(c) + + t.SetupTool(&Tool{Name: "tool1", Predefined: true}) + t.SetupVartypes() + t.SetupFileLines("Makefile", + MkRcsID, + "", + "USE_TOOLS.NetBSD+=\ttool1") + + G.CurrentDir = t.TmpDir() + CheckdirToplevel() + + // No error about "Unknown tool \"NetBSD\"." + t.CheckOutputEmpty() +} --_----------=_152495837221680--