Content-Transfer-Encoding: 7bit
Content-Type: multipart/mixed; boundary="_----------=_1473195261212190"
MIME-Version: 1.0
Date: Tue, 6 Sep 2016 20:54:21 +0000
From: "Roland Illig" <rillig@netbsd.org>
Subject: CVS commit: pkgsrc/pkgtools/pkglint/files
To: pkgsrc-changes@NetBSD.org
Reply-To: rillig@netbsd.org
Message-Id: <20160906205421.335EDFBD1@cvs.NetBSD.org>
Sender: pkgsrc-changes-owner@NetBSD.org
Precedence: bulk

This is a multi-part message in MIME format.

--_----------=_1473195261212190
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
Content-Type: text/plain; charset="US-ASCII"

Module Name:	pkgsrc
Committed By:	rillig
Date:		Tue Sep  6 20:54:21 UTC 2016

Modified Files:
	pkgsrc/pkgtools/pkglint/files: globaldata.go mklines.go mklines_test.go
	    shell.go vardefs.go vartypecheck.go

Log Message:
Allow packages to define their own tools using TOOLS_CREATE.

Suggested by @jperkin for lang/rust.


To generate a diff of this commit:
cvs rdiff -u -r1.13 -r1.14 pkgsrc/pkgtools/pkglint/files/globaldata.go
cvs rdiff -u -r1.8 -r1.9 pkgsrc/pkgtools/pkglint/files/mklines.go
cvs rdiff -u -r1.7 -r1.8 pkgsrc/pkgtools/pkglint/files/mklines_test.go
cvs rdiff -u -r1.12 -r1.13 pkgsrc/pkgtools/pkglint/files/shell.go
cvs rdiff -u -r1.15 -r1.16 pkgsrc/pkgtools/pkglint/files/vardefs.go
cvs rdiff -u -r1.18 -r1.19 pkgsrc/pkgtools/pkglint/files/vartypecheck.go

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.


--_----------=_1473195261212190
Content-Disposition: inline
Content-Length: 10287
Content-Transfer-Encoding: binary
Content-Type: text/x-diff; charset=us-ascii

Modified files:

Index: pkgsrc/pkgtools/pkglint/files/globaldata.go
diff -u pkgsrc/pkgtools/pkglint/files/globaldata.go:1.13 pkgsrc/pkgtools/pkglint/files/globaldata.go:1.14
--- pkgsrc/pkgtools/pkglint/files/globaldata.go:1.13	Sun Jul 10 21:24:47 2016
+++ pkgsrc/pkgtools/pkglint/files/globaldata.go	Tue Sep  6 20:54:21 2016
@@ -138,23 +138,7 @@ func (gd *GlobalData) loadTools() {
 		fname := G.globalData.Pkgsrcdir + "/mk/tools/" + basename
 		lines := LoadExistingLines(fname, true)
 		for _, line := range lines {
-			if m, varname, _, _, _, value, _ := MatchVarassign(line.Text); m {
-				if varname == "TOOLS_CREATE" && (value == "[" || matches(value, `^?[-\w.]+$`)) {
-					reg.Register(value)
-
-				} else if m, toolname := match1(varname, `^_TOOLS_VARNAME\.([-\w.]+|\[)$`); m {
-					reg.RegisterVarname(toolname, value)
-
-				} else if m, toolname := match1(varname, `^(?:TOOLS_PATH|_TOOLS_DEPMETHOD)\.([-\w.]+|\[)$`); m {
-					reg.Register(toolname)
-
-				} else if m, toolname := match1(varname, `_TOOLS\.(.*)`); m {
-					reg.Register(toolname)
-					for _, tool := range splitOnSpace(value) {
-						reg.Register(tool)
-					}
-				}
-			}
+			reg.ParseToolLine(line)
 		}
 	}
 
@@ -580,3 +564,23 @@ func (tr *ToolRegistry) Trace() {
 		traceStep("tool %+v", tr.byName[toolname])
 	}
 }
+
+func (tr *ToolRegistry) ParseToolLine(line *Line) {
+	if m, varname, _, _, _, value, _ := MatchVarassign(line.Text); m {
+		if varname == "TOOLS_CREATE" && (value == "[" || matches(value, `^?[-\w.]+$`)) {
+			tr.Register(value)
+
+		} else if m, toolname := match1(varname, `^_TOOLS_VARNAME\.([-\w.]+|\[)$`); m {
+			tr.RegisterVarname(toolname, value)
+
+		} else if m, toolname := match1(varname, `^(?:TOOLS_PATH|_TOOLS_DEPMETHOD)\.([-\w.]+|\[)$`); m {
+			tr.Register(toolname)
+
+		} else if m, toolname := match1(varname, `_TOOLS\.(.*)`); m {
+			tr.Register(toolname)
+			for _, tool := range splitOnSpace(value) {
+				tr.Register(tool)
+			}
+		}
+	}
+}

Index: pkgsrc/pkgtools/pkglint/files/mklines.go
diff -u pkgsrc/pkgtools/pkglint/files/mklines.go:1.8 pkgsrc/pkgtools/pkglint/files/mklines.go:1.9
--- pkgsrc/pkgtools/pkglint/files/mklines.go:1.8	Sun Jul 10 21:24:47 2016
+++ pkgsrc/pkgtools/pkglint/files/mklines.go	Tue Sep  6 20:54:21 2016
@@ -16,6 +16,7 @@ type MkLines struct {
 	buildDefs      map[string]bool    // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it.
 	plistVars      map[string]bool    // Variables that are registered in PLIST_VARS, to ensure that all user-defined variables are added to it.
 	tools          map[string]bool    // Set of tools that are declared to be used.
+	toolRegistry   ToolRegistry       // Tools defined in file scope.
 	SeenBsdPrefsMk bool
 	indentation    Indentation // Indentation depth of preprocessing directives
 }
@@ -42,6 +43,7 @@ func NewMkLines(lines []*Line) *MkLines 
 		make(map[string]bool),
 		make(map[string]bool),
 		tools,
+		NewToolRegistry(),
 		false,
 		Indentation{}}
 }
@@ -202,6 +204,8 @@ func (mklines *MkLines) determineDefined
 				defineVar(mkline, osvar)
 			}
 		}
+
+		mklines.toolRegistry.ParseToolLine(mkline.Line)
 	}
 }
 

Index: pkgsrc/pkgtools/pkglint/files/mklines_test.go
diff -u pkgsrc/pkgtools/pkglint/files/mklines_test.go:1.7 pkgsrc/pkgtools/pkglint/files/mklines_test.go:1.8
--- pkgsrc/pkgtools/pkglint/files/mklines_test.go:1.7	Sat Jul  9 09:43:48 2016
+++ pkgsrc/pkgtools/pkglint/files/mklines_test.go	Tue Sep  6 20:54:21 2016
@@ -386,3 +386,30 @@ func (s *Suite) Test_MkLines_DetermineUs
 	c.Check(mklines.varuse["outer."], equals, mkline)
 	c.Check(mklines.varuse["outer.*"], equals, mkline)
 }
+
+func (s *Suite) Test_MkLines_PrivateTool_Undefined(c *check.C) {
+	G.globalData.InitVartypes()
+	s.UseCommandLine(c, "-Wall")
+	mklines := s.NewMkLines("fname",
+		mkrcsid,
+		"",
+		"\tmd5sum filename")
+
+	mklines.Check()
+
+	c.Check(s.Output(), equals, "WARN: fname:3: Unknown shell command \"md5sum\".\n")
+}
+
+func (s *Suite) Test_MkLines_PrivateTool_Defined(c *check.C) {
+	G.globalData.InitVartypes()
+	s.UseCommandLine(c, "-Wall")
+	mklines := s.NewMkLines("fname",
+		mkrcsid,
+		"TOOLS_CREATE+=\tmd5sum",
+		"",
+		"\tmd5sum filename")
+
+	mklines.Check()
+
+	c.Check(s.Output(), equals, "")
+}

Index: pkgsrc/pkgtools/pkglint/files/shell.go
diff -u pkgsrc/pkgtools/pkglint/files/shell.go:1.12 pkgsrc/pkgtools/pkglint/files/shell.go:1.13
--- pkgsrc/pkgtools/pkglint/files/shell.go:1.12	Sun Jul 10 21:24:47 2016
+++ pkgsrc/pkgtools/pkglint/files/shell.go	Tue Sep  6 20:54:21 2016
@@ -474,12 +474,15 @@ func (scc *SimpleCommandChecker) handleT
 	}
 
 	shellword := scc.strcmd.Name
-	tool := G.globalData.Tools.byName[shellword]
+	tool, localTool := G.globalData.Tools.byName[shellword], false
+	if tool == nil && G.Mk != nil {
+		tool, localTool = G.Mk.toolRegistry.byName[shellword], true
+	}
 	if tool == nil {
 		return false
 	}
 
-	if !G.Mk.tools[shellword] && !G.Mk.tools["g"+shellword] {
+	if !localTool && !G.Mk.tools[shellword] && !G.Mk.tools["g"+shellword] {
 		scc.shline.line.Warn1("The %q tool is used but not added to USE_TOOLS.", shellword)
 	}
 

Index: pkgsrc/pkgtools/pkglint/files/vardefs.go
diff -u pkgsrc/pkgtools/pkglint/files/vardefs.go:1.15 pkgsrc/pkgtools/pkglint/files/vardefs.go:1.16
--- pkgsrc/pkgtools/pkglint/files/vardefs.go:1.15	Tue Sep  6 19:23:03 2016
+++ pkgsrc/pkgtools/pkglint/files/vardefs.go	Tue Sep  6 20:54:21 2016
@@ -17,6 +17,35 @@ import (
 // Last synced with mk/defaults/mk.conf revision 1.118
 
 func (gd *GlobalData) InitVartypes() {
+
+	// A package-defined variable may be set in all Makefiles except buildlink3.mk and builtin.mk.
+	pkg := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "Makefile: set, use; buildlink3.mk, builtin.mk:; Makefile.*, *.mk: default, set, use")
+	}
+
+	// A package-defined list may be appended to in all Makefiles except buildlink3.mk and builtin.mk.
+	// Simple assignment (instead of appending) is only allowed in Makefile and Makefile.common.
+	pkglist := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "Makefile, Makefile.common, options.mk: append, default, set, use; buildlink3.mk, builtin.mk:; *.mk: append, default, use")
+	}
+
+	// A user-defined or system-defined variable must not be set by any
+	// package file. It also must not be used in buildlink3.mk and
+	// builtin.mk files or at load-time, since the system/user preferences
+	// may not have been loaded when these files are included.
+	sys := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "buildlink3.mk:; *: use")
+	}
+	usr := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "buildlink3.mk:; *: use-loadtime, use")
+	}
+	bl3list := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: append")
+	}
+	cmdline := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use")
+	}
+
 	usr("ALLOW_VULNERABLE_PACKAGES", lkNone, BtYes)
 	usr("MANINSTALL", lkShell, enum("maninstall catinstall"))
 	usr("MANZ", lkNone, BtYes)
@@ -688,7 +717,7 @@ func (gd *GlobalData) InitVartypes() {
 	sys("TOOLS_ALIASES", lkShell, BtFilename)
 	sys("TOOLS_BROKEN", lkShell, BtTool)
 	sys("TOOLS_CMD.*", lkNone, BtPathname)
-	sys("TOOLS_CREATE", lkShell, BtTool)
+	acl("TOOLS_CREATE", lkShell, BtTool, "Makefile, Makefile.common, options.mk: append")
 	acl("TOOLS_DEPENDS.*", lkSpace, BtDependencyWithPath, "buildlink3.mk:; Makefile, Makefile.*: set, default; *: use")
 	sys("TOOLS_GNU_MISSING", lkShell, BtTool)
 	sys("TOOLS_NOOP", lkShell, BtTool)
@@ -841,31 +870,3 @@ func parseAclEntries(varname string, acl
 	}
 	return result
 }
-
-// A package-defined variable may be set in all Makefiles except buildlink3.mk and builtin.mk.
-func pkg(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "Makefile: set, use; buildlink3.mk, builtin.mk:; Makefile.*, *.mk: default, set, use")
-}
-
-// A package-defined list may be appended to in all Makefiles except buildlink3.mk and builtin.mk.
-// Simple assignment (instead of appending) is only allowed in Makefile and Makefile.common.
-func pkglist(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "Makefile, Makefile.common, options.mk: append, default, set, use; buildlink3.mk, builtin.mk:; *.mk: append, default, use")
-}
-
-// A user-defined or system-defined variable must not be set by any
-// package file. It also must not be used in buildlink3.mk and
-// builtin.mk files or at load-time, since the system/user preferences
-// may not have been loaded when these files are included.
-func sys(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "buildlink3.mk:; *: use")
-}
-func usr(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "buildlink3.mk:; *: use-loadtime, use")
-}
-func bl3list(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: append")
-}
-func cmdline(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use")
-}

Index: pkgsrc/pkgtools/pkglint/files/vartypecheck.go
diff -u pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.18 pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.19
--- pkgsrc/pkgtools/pkglint/files/vartypecheck.go:1.18	Sun Jul 10 21:24:47 2016
+++ pkgsrc/pkgtools/pkglint/files/vartypecheck.go	Tue Sep  6 20:54:21 2016
@@ -894,7 +894,7 @@ func (cv *VartypeCheck) Tool() {
 		// no warning for package-defined tool definitions
 
 	} else if m, toolname, tooldep := match2(cv.Value, `^([-\w]+|\[)(?::(\w+))?$`); m {
-		if G.globalData.Tools.byName[toolname] == nil {
+		if G.globalData.Tools.byName[toolname] == nil && (G.Mk == nil || G.Mk.toolRegistry.byName[toolname] == nil) {
 			cv.Line.Error1("Unknown tool %q.", toolname)
 		}
 		switch tooldep {


--_----------=_1473195261212190--