@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.586 2019/06/30 20:56:18 rillig Exp $
+# $NetBSD: Makefile,v 1.587 2019/07/01 22:25:52 rillig Exp $
 
-PKGNAME=	pkglint-5.7.14
+PKGNAME=	pkglint-5.7.15
 CATEGORIES=	pkgtools
 DISTNAME=	tools
 MASTER_SITES=	${MASTER_SITE_GITHUB:=golang/}

@@ -52,9 +52,6 @@
 const AutofixFormat = "AutofixFormat"
 
 func NewAutofix(line *Line) *Autofix {
-	// FIXME: replacing the returned value with
-	//  &Autofix{line: line, autofixShortTerm: autofixShortTerm{anyway: true}}
-	//  makes some tests output source code without diagnostic.
 	return &Autofix{line: line}
 }
 

@@ -988,6 +988,31 @@
 	t.CheckOutputEmpty()
 }
 
+func (s *Suite) Test_Autofix_Apply__source_autofix_no_change(c *check.C) {
+	t := s.Init(c)
+
+	t.SetUpCommandLine("--autofix", "--source")
+	lines := t.SetUpFileLines("filename",
+		"word word word")
+
+	fix := lines.Lines[0].Autofix()
+	fix.Notef("Word should be replaced, but pkglint is not sure which one.")
+	fix.Replace("word", "replacement")
+	fix.Anyway()
+	fix.Apply()
+
+	lines.SaveAutofixChanges()
+
+	// Nothing is replaced since, as of June 2019, pkglint doesn't
+	// know which of the three "word" should be replaced.
+	//
+	// The note is not logged since fix.Anyway only applies when neither
+	// --show-autofix nor --autofix is given in the command line.
+	t.CheckOutputEmpty()
+	t.CheckFileLines("filename",
+		"word word word")
+}
+
 // Ensures that without explanations, the separator between the individual
 // diagnostics are generated.
 func (s *Suite) Test_Autofix_Apply__source_without_explain(c *check.C) {

@@ -259,6 +259,45 @@
 	t.Check(G.Pkgsrc.LastChange["pkgpath"].Action, equals, Moved)
 }
 
+func (s *Suite) Test_Pkgsrc_checkRemovedAfterLastFreeze(c *check.C) {
+	t := s.Init(c)
+
+	t.SetUpCommandLine("-Wall", "--source")
+	t.SetUpPkgsrc()
+	t.CreateFileLines("doc/CHANGES-2019",
+		CvsID,
+		"",
+		"\tUpdated category/updated-before to 1.0 [updater 2019-04-01]",
+		"\tmk/bsd.pkg.mk: started freeze for pkgsrc-2019Q1 branch [freezer 2019-06-21]",
+		"\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2019Q1 branch [freezer 2019-06-25]",
+		"\tUpdated category/updated-after to 1.0 [updater 2019-07-01]",
+		"\tAdded category/added-after version 1.0 [updater 2019-07-01]",
+		"\tMoved category/moved-from to category/moved-to [author 2019-07-02]",
+		"\tDowngraded category/downgraded to 1.0 [author 2019-07-03]")
+	t.FinishSetUp()
+
+	// It doesn't matter whether the last visible package change was before
+	// or after the latest freeze. The crucial point is that the most
+	// interesting change is the invisible one, which is the removal.
+	// And for finding the removal reliably, it doesn't matter how long ago
+	// the last package change was.
+
+	// The empty lines in the following output demonstrate the cheating
+	// by creating fake lines from Change.Location.
+	t.CheckOutputLines(
+		"ERROR: ~/doc/CHANGES-2019:3: Package category/updated-before "+
+			"must either exist or be marked as removed.",
+		"",
+		"ERROR: ~/doc/CHANGES-2019:6: Package category/updated-after "+
+			"must either exist or be marked as removed.",
+		"",
+		"ERROR: ~/doc/CHANGES-2019:7: Package category/added-after "+
+			"must either exist or be marked as removed.",
+		"",
+		"ERROR: ~/doc/CHANGES-2019:9: Package category/downgraded "+
+			"must either exist or be marked as removed.")
+}
+
 func (s *Suite) Test_Pkgsrc_loadDocChanges__not_found(c *check.C) {
 	t := s.Init(c)
 
@@ -925,7 +964,7 @@
 		"\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25]")
 	t.FinishSetUp()
 
-	t.Check(G.Pkgsrc.FreezeStart, equals, "2018-03-25")
+	t.Check(G.Pkgsrc.LastFreezeStart, equals, "2018-03-25")
 }
 
 func (s *Suite) Test_Pkgsrc__not_frozen(c *check.C) {
@@ -937,7 +976,8 @@
 		"\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2018Q2 branch [freezer 2018-03-27]")
 	t.FinishSetUp()
 
-	t.Check(G.Pkgsrc.FreezeStart, equals, "")
+	t.Check(G.Pkgsrc.LastFreezeStart, equals, "2018-03-25")
+	t.Check(G.Pkgsrc.LastFreezeEnd, equals, "2018-03-27")
 }
 
 func (s *Suite) Test_Pkgsrc__frozen_with_typo(c *check.C) {
@@ -949,7 +989,7 @@
 		"\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25")
 	t.FinishSetUp()
 
-	t.Check(G.Pkgsrc.FreezeStart, equals, "")
+	t.Check(G.Pkgsrc.LastFreezeStart, equals, "")
 }
 
 func (s *Suite) Test_Change_Version(c *check.C) {

@@ -48,7 +48,7 @@
 	}
 
 	if !llex.EOF() {
-		if m := G.res.Match(llex.lines.Lines[llex.index].Text, re); m != nil {
+		if m := match(llex.lines.Lines[llex.index].Text, re); m != nil {
 			llex.index++
 			return m
 		}

@@ -451,7 +451,7 @@
 				break
 			}
 			if hasSuffix(varname, ".") {
-				if !parser.lexer.SkipRegexp(G.res.Compile(`^<\w+>`)) {
+				if !parser.lexer.SkipRegexp(regcomp(`^<\w+>`)) {
 					break
 				}
 				varname += "*"

@@ -251,7 +251,7 @@
 
 	case '=', 'D', 'M', 'N', 'U':
 		lexer.Skip(1)
-		re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:\\}]|\$\$|\\.)+`, `^([^$:\\)]|\$\$|\\.)+`)))
+		re := regcomp(regex.Pattern(ifelseStr(closing == '}', `^([^$:\\}]|\$\$|\\.)+`, `^([^$:\\)]|\$\$|\\.)+`)))
 		for p.VarUse() != nil || lexer.SkipRegexp(re) {
 		}
 		arg := lexer.Since(mark)
@@ -268,7 +268,7 @@
 		}
 
 	case '[':
-		if lexer.SkipRegexp(G.res.Compile(`^\[(?:[-.\d]+|#)\]`)) {
+		if lexer.SkipRegexp(regcomp(`^\[(?:[-.\d]+|#)\]`)) {
 			return lexer.Since(mark)
 		}
 
@@ -283,7 +283,7 @@
 
 	lexer.Reset(mark)
 
-	re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^:$}]|\$\$)+`, `^([^:$)]|\$\$)+`)))
+	re := regcomp(regex.Pattern(ifelseStr(closing == '}', `^([^:$}]|\$\$)+`, `^([^:$)]|\$\$)+`)))
 	for p.VarUse() != nil || lexer.SkipRegexp(re) {
 	}
 	modifier := lexer.Since(mark)
@@ -309,7 +309,7 @@
 func (p *MkParser) varUseText(closing byte) string {
 	lexer := p.lexer
 	start := lexer.Mark()
-	re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:}]|\$\$)+`, `^([^$:)]|\$\$)+`)))
+	re := regcomp(regex.Pattern(ifelseStr(closing == '}', `^([^$:}]|\$\$)+`, `^([^$:)]|\$\$)+`)))
 	for p.VarUse() != nil || lexer.SkipRegexp(re) {
 	}
 	return lexer.Since(start)
@@ -391,7 +391,7 @@
 		return false
 	}
 
-	re := G.res.Compile(`^([^$@\\]|\\.)+`)
+	re := regcomp(`^([^$@\\]|\\.)+`)
 	for p.VarUse() != nil || lexer.SkipString("$$") || lexer.SkipRegexp(re) {
 	}
 
@@ -498,11 +498,11 @@
 		if lhs != nil {
 			lexer.SkipHspace()
 
-			if m := lexer.NextRegexp(G.res.Compile(`^(<|<=|==|!=|>=|>)[\t ]*(0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil {
+			if m := lexer.NextRegexp(regcomp(`^(<|<=|==|!=|>=|>)[\t ]*(0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil {
 				return &MkCond{CompareVarNum: &MkCondCompareVarNum{lhs, m[1], m[2]}}
 			}
 
-			m := lexer.NextRegexp(G.res.Compile(`^(?:<|<=|==|!=|>=|>)`))
+			m := lexer.NextRegexp(regcomp(`^(?:<|<=|==|!=|>=|>)`))
 			if m == nil {
 				return &MkCond{Var: lhs} // See devel/bmake/files/cond.c:/\* For \.if \$/
 			}
@@ -510,7 +510,7 @@
 
 			op := m[0]
 			if op == "==" || op == "!=" {
-				if mrhs := lexer.NextRegexp(G.res.Compile(`^"([^"\$\\]*)"`)); mrhs != nil {
+				if mrhs := lexer.NextRegexp(regcomp(`^"([^"\$\\]*)"`)); mrhs != nil {
 					return &MkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, mrhs[1]}}
 				}
 			}
@@ -559,7 +559,7 @@
 		}
 
 		// See devel/bmake/files/cond.c:/^CondCvtArg
-		if m := lexer.NextRegexp(G.res.Compile(`^(?:0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil {
+		if m := lexer.NextRegexp(regcomp(`^(?:0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil {
 			return &MkCond{Num: m[0]}
 		}
 	}
@@ -652,8 +652,8 @@
 
 	for {
 		if p.VarUse() != nil ||
-			lexer.SkipRegexp(G.res.Compile(`^[\w.*+,{}]+`)) ||
-			lexer.SkipRegexp(G.res.Compile(`^\[[\w-]+\]`)) {
+			lexer.SkipRegexp(regcomp(`^[\w.*+,{}]+`)) ||
+			lexer.SkipRegexp(regcomp(`^\[[\w-]+\]`)) {
 			continue
 		}
 
@@ -696,7 +696,7 @@
 			return lexer.Since(mark)
 		}
 
-		m := lexer.NextRegexp(G.res.Compile(`^\d[\w.]*`))
+		m := lexer.NextRegexp(regcomp(`^\d[\w.]*`))
 		if m != nil {
 			return m[0]
 		}
@@ -749,7 +749,7 @@
 	if lexer.SkipByte('-') && lexer.Rest() != "" {
 		versionMark := lexer.Mark()
 
-		for p.VarUse() != nil || lexer.SkipRegexp(G.res.Compile(`^[\w\[\]*_.\-]+`)) {
+		for p.VarUse() != nil || lexer.SkipRegexp(regcomp(`^[\w\[\]*_.\-]+`)) {
 		}
 
 		if !lexer.SkipString("{,nb*}") {

@@ -1214,8 +1214,8 @@
 }
 
 func (pkg *Package) checkFreeze(filename string) {
-	freezeStart := G.Pkgsrc.FreezeStart
-	if freezeStart == "" {
+	freezeStart := G.Pkgsrc.LastFreezeStart
+	if freezeStart == "" || G.Pkgsrc.LastFreezeEnd != "" {
 		return
 	}
 

@@ -962,7 +962,7 @@
 		"ERROR: gnu-style.mk:1: Unknown Makefile line format: \"ifeq ($(CC),gcc)\".",
 		"ERROR: gnu-style.mk:3: Unknown Makefile line format: \"else\".",
 		"ERROR: gnu-style.mk:5: Unknown Makefile line format: \"endif\".",
-		"ERROR: gnu-style.mk:1: Expected \"# $NetBSD: package_test.go,v 1.48 2019/06/30 20:56:19 rillig Exp $\".",
+		"ERROR: gnu-style.mk:1: Expected \""+MkCvsID+"\".",
 		"WARN: gnu-style.mk:2: IS_GCC is defined but not used.",
 
 		// There is no warning about files/gnu-style.mk since pkglint
@@ -983,7 +983,7 @@
 		"ERROR: ../../category/other/gnu-style.mk:1: Unknown Makefile line format: \"ifeq ($(CC),gcc)\".",
 		"ERROR: ../../category/other/gnu-style.mk:3: Unknown Makefile line format: \"else\".",
 		"ERROR: ../../category/other/gnu-style.mk:5: Unknown Makefile line format: \"endif\".",
-		"ERROR: ../../category/other/gnu-style.mk:1: Expected \"# $NetBSD: package_test.go,v 1.48 2019/06/30 20:56:19 rillig Exp $\".",
+		"ERROR: ../../category/other/gnu-style.mk:1: Expected \""+MkCvsID+"\".",
 		"WARN: ../../category/other/gnu-style.mk:2: IS_GCC is defined but not used.",
 
 		"ERROR: patches/patch-Makefile.mk: Contains no patch.",

@@ -246,7 +246,7 @@
 	exitcode := t.Main("-Wall", "--autofix", t.File("filename.mk"))
 
 	t.CheckOutputLines(
-		"AUTOFIX: ~/filename.mk:1: Inserting a line \"# $NetBSD: pkglint_test.go,v 1.44 2019/06/30 20:56:19 rillig Exp $\" before this line.")
+		"AUTOFIX: ~/filename.mk:1: Inserting a line \"" + MkCvsID + "\" before this line.")
 	t.Check(exitcode, equals, 0)
 }
 

@@ -36,8 +36,9 @@
 	suggestedUpdates    []SuggestedUpdate
 	suggestedWipUpdates []SuggestedUpdate
 
-	LastChange  map[string]*Change
-	FreezeStart string // e.g. "2018-01-01", or ""
+	LastChange      map[string]*Change
+	LastFreezeStart string // e.g. "2018-01-01", or ""
+	LastFreezeEnd   string // e.g. "2018-01-01", or ""
 
 	listVersions map[string][]string // See Pkgsrc.ListVersions
 
@@ -64,6 +65,7 @@
 		nil,
 		make(map[string]*Change),
 		"",
+		"",
 		make(map[string][]string),
 		NewScope(),
 		make(map[string]string),
@@ -517,11 +519,14 @@
 		if hasPrefix(line.Text, "\tmk/") {
 			infra = true
 			if hasPrefix(line.Text, "\tmk/bsd.pkg.mk: started freeze for") {
-				if m, freezeDate := match1(line.Text, `(\d\d\d\d-\d\d-\d\d)\]$`); m {
-					src.FreezeStart = freezeDate
+				if m, date := match1(line.Text, `(\d\d\d\d-\d\d-\d\d)\]$`); m {
+					src.LastFreezeStart = date
+					src.LastFreezeEnd = ""
 				}
 			} else if hasPrefix(line.Text, "\tmk/bsd.pkg.mk: freeze ended for") {
-				src.FreezeStart = ""
+				if m, date := match1(line.Text, `(\d\d\d\d-\d\d-\d\d)\]$`); m {
+					src.LastFreezeEnd = date
+				}
 			}
 		}
 		if infra {
@@ -602,6 +607,40 @@
 				src.LastChange[change.Target()] = change
 			}
 		}
+	}
+
+	src.checkRemovedAfterLastFreeze()
+}
+
+func (src *Pkgsrc) checkRemovedAfterLastFreeze() {
+	if src.LastFreezeStart == "" || G.Wip {
+		return
+	}
+
+	var wrong []*Change
+	for pkgpath, change := range src.LastChange {
+		switch change.Action {
+		case Added, Updated, Downgraded:
+			if !dirExists(src.File(pkgpath)) {
+				wrong = append(wrong, change)
+			}
+		}
+	}
+
+	sort.Slice(wrong, func(i, j int) bool {
+		ei, ej := wrong[i], wrong[j]
+		if ei.Date != ej.Date {
+			return ei.Date < ej.Date
+		}
+		return ei.Location.firstLine < ej.Location.firstLine
+	})
+
+	for _, change := range wrong {
+		// It's a bit cheated to construct a Line from only a Location,
+		// without the wrong text. That's only because I'm too lazy loading
+		// the file again, and the original text is not lying around anywhere.
+		line := NewLineMulti(change.Location.Filename, int(change.Location.firstLine), int(change.Location.lastLine), "", nil)
+		line.Errorf("Package %s must either exist or be marked as removed.", change.Pkgpath)
 	}
 }
 

@@ -137,7 +137,7 @@
 		return &ShAtom{shtText, lexer.Since(mark), shqBacktSquot, nil}
 	case lexer.NextHspace() != "":
 		return &ShAtom{shtSpace, lexer.Since(mark), q, nil}
-	case lexer.SkipRegexp(G.res.Compile("^#[^`]*")):
+	case lexer.SkipRegexp(regcomp("^#[^`]*")):
 		return &ShAtom{shtComment, lexer.Since(mark), q, nil}
 	}
 	return p.shAtomInternal(q, false, false)
@@ -158,7 +158,7 @@
 		return &ShAtom{shtText, lexer.Since(mark), shqSubshSquot, nil}
 	case lexer.SkipByte('`'):
 		return &ShAtom{shtText, lexer.Since(mark), shqSubshBackt, nil}
-	case lexer.SkipRegexp(G.res.Compile(`^#[^)]*`)):
+	case lexer.SkipRegexp(regcomp(`^#[^)]*`)):
 		return &ShAtom{shtComment, lexer.Since(mark), q, nil}
 	case lexer.SkipByte(')'):
 		// The closing parenthesis can have multiple meanings:
@@ -191,7 +191,7 @@
 		return &ShAtom{shtText, lexer.Since(mark), shqDquotBacktDquot, nil}
 	case lexer.SkipByte('\''):
 		return &ShAtom{shtText, lexer.Since(mark), shqDquotBacktSquot, nil}
-	case lexer.SkipRegexp(G.res.Compile("^#[^`]*")):
+	case lexer.SkipRegexp(regcomp("^#[^`]*")):
 		return &ShAtom{shtComment, lexer.Since(mark), q, nil}
 	case lexer.NextHspace() != "":
 		return &ShAtom{shtSpace, lexer.Since(mark), q, nil}
@@ -300,13 +300,13 @@
 		_ = `^[\t "$&'();<>\\|]+` // These are not allowed in shqPlain.
 
 		switch {
-		case lexer.SkipRegexp(G.res.Compile(`^[!#%*+,\-./0-9:=?@A-Z\[\]^_a-z{}~]+`)):
+		case lexer.SkipRegexp(regcomp(`^[!#%*+,\-./0-9:=?@A-Z\[\]^_a-z{}~]+`)):
 			break
-		case dquot && lexer.SkipRegexp(G.res.Compile(`^[\t &'();<>|]+`)):
+		case dquot && lexer.SkipRegexp(regcomp(`^[\t &'();<>|]+`)):
 			break
 		case squot && lexer.SkipByte('`'):
 			break
-		case squot && lexer.SkipRegexp(G.res.Compile(`^[\t "&();<>\\|]+`)):
+		case squot && lexer.SkipRegexp(regcomp(`^[\t "&();<>\\|]+`)):
 			break
 		case squot && lexer.SkipString("$$"):
 			break
@@ -314,7 +314,7 @@
 			break loop
 		case lexer.SkipString("\\$$"):
 			break
-		case lexer.SkipRegexp(G.res.Compile(`^\\[^$]`)):
+		case lexer.SkipRegexp(regcomp(`^\\[^$]`)):
 			break
 		case matches(lexer.Rest(), `^\$\$[^!#(*\-0-9?@A-Z_a-z{]`):
 			lexer.NextString("$$")
@@ -353,7 +353,7 @@
 	brace := lexer.SkipByte('{')
 
 	varnameStart := lexer.Mark()
-	if !lexer.SkipRegexp(G.res.Compile(`^(?:[!#*\-?@]|\$\$|[A-Za-z_]\w*|\d+)`)) {
+	if !lexer.SkipRegexp(regcomp(`^(?:[!#*\-?@]|\$\$|[A-Za-z_]\w*|\d+)`)) {
 		lexer.Reset(beforeDollar)
 		return nil
 	}
@@ -364,7 +364,7 @@
 	}
 
 	if brace {
-		lexer.SkipRegexp(G.res.Compile(`^(?:##?|%%?|:?[+\-=?])[^$\\{}]*`))
+		lexer.SkipRegexp(regcomp(`^(?:##?|%%?|:?[+\-=?])[^$\\{}]*`))
 		if !lexer.SkipByte('}') {
 			lexer.Reset(beforeDollar)
 			return nil
@@ -388,7 +388,7 @@
 		lexer.SkipByte('|'),
 		lexer.SkipByte('&'):
 		return &ShAtom{shtOperator, lexer.Since(mark), q, nil}
-	case lexer.SkipRegexp(G.res.Compile(`^\d*(?:<<-|<<|<&|<>|>>|>&|>\||<|>)`)):
+	case lexer.SkipRegexp(regcomp(`^\d*(?:<<-|<<|<&|<>|>>|>&|>\||<|>)`)):
 		return &ShAtom{shtOperator, lexer.Since(mark), q, nil}
 	}
 	return nil

@@ -694,7 +694,7 @@
 			"with \"SUBST_VARS.pfx+=\\tPREFIX\".")
 
 	t.CheckFileLinesDetab("subst.mk",
-		"# $NetBSD: substcontext_test.go,v 1.27 2019/06/30 20:56:19 rillig Exp $",
+		MkCvsID,
 		"",
 		"SUBST_CLASSES+=         pfx",
 		"SUBST_STAGE.pfx=        pre-configure",
@@ -733,7 +733,7 @@
 			"with \"SUBST_VARS.pfx=\\t\\tPREFIX\".")
 
 	t.CheckFileLinesDetab("subst.mk",
-		"# $NetBSD: substcontext_test.go,v 1.27 2019/06/30 20:56:19 rillig Exp $",
+		MkCvsID,
 		"",
 		"SUBST_CLASSES+=         pfx",
 		"SUBST_STAGE.pfx=        pre-configure",
@@ -767,7 +767,7 @@
 			"with \"SUBST_VARS.id=\\tPREFIX\".")
 
 	t.CheckFileLinesDetab("subst.mk",
-		"# $NetBSD: substcontext_test.go,v 1.27 2019/06/30 20:56:19 rillig Exp $",
+		MkCvsID,
 		"",
 		"SUBST_CLASSES+= id",
 		"SUBST_STAGE.id= pre-configure",
@@ -801,7 +801,7 @@
 			"with \"SUBST_VARS.fix-paths=\\t\\tPREFIX\".")
 
 	t.CheckFileLinesDetab("subst.mk",
-		"# $NetBSD: substcontext_test.go,v 1.27 2019/06/30 20:56:19 rillig Exp $",
+		MkCvsID,
 		"",
 		"SUBST_CLASSES+=                 fix-paths",
 		"SUBST_STAGE.fix-paths=          pre-configure",

@@ -42,6 +42,12 @@
 func sprintf(format string, args ...interface{}) string {
 	return fmt.Sprintf(format, args...)
 }
+func regcomp(re regex.Pattern) *regexp.Regexp {
+	return G.res.Compile(re)
+}
+func match(s string, re regex.Pattern) []string {
+	return G.res.Match(s, re)
+}
 func matches(s string, re regex.Pattern) bool {
 	return G.res.Matches(s, re)
 }