Sun Feb 12 04:12:54 2023 UTC ()
pbulk-base-0.56: Support for adjusting scheduling

Switch to a weighted scheduling algorithm. Before, build order was based
on number of reachable nodes in the dependency graph. This assumes that
heavy packages are required by other packages. Some big packages
nowadays violate that assumption and can result in long periods at the
end of a build where only a few builders are active. Annotating those
packages with PBULK_WEIGHT in the pbulk-index output can boost their
priority to let them be built earlier. The default weight is 100.

Note: the pbulk-build report has grown an extra field per line with the
computed effective weight of each package. This file is normally used
only internally.


(joerg)
diff -r1.26 -r1.27 pkgsrc/pkgtools/pbulk-base/Makefile
diff -r1.18 -r1.19 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c
diff -r1.8 -r1.9 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h

cvs diff -r1.26 -r1.27 pkgsrc/pkgtools/pbulk-base/Makefile (expand / switch to unified diff)

--- pkgsrc/pkgtools/pbulk-base/Makefile 2023/02/10 23:14:32 1.26
+++ pkgsrc/pkgtools/pbulk-base/Makefile 2023/02/12 04:12:54 1.27
@@ -1,16 +1,16 @@ @@ -1,16 +1,16 @@
1# $NetBSD: Makefile,v 1.26 2023/02/10 23:14:32 joerg Exp $ 1# $NetBSD: Makefile,v 1.27 2023/02/12 04:12:54 joerg Exp $
2 2
3DISTNAME= pbulk-base-0.55 3DISTNAME= pbulk-base-0.56
4COMMENT= Core components of the modular bulk build framework 4COMMENT= Core components of the modular bulk build framework
5 5
6.include "../../pkgtools/pbulk/Makefile.common" 6.include "../../pkgtools/pbulk/Makefile.common"
7 7
8USE_FEATURES= nbcompat 8USE_FEATURES= nbcompat
9USE_TOOLS+= nroff 9USE_TOOLS+= nroff
10 10
11INSTALLATION_DIRS= bin ${PKGMANDIR}/cat1 ${PKGMANDIR}/man1 11INSTALLATION_DIRS= bin ${PKGMANDIR}/cat1 ${PKGMANDIR}/man1
12USE_BSD_MAKEFILE= yes 12USE_BSD_MAKEFILE= yes
13 13
14MAKE_FLAGS+= CWARNFLAGS.clang=-Wno-error=missing-noreturn 14MAKE_FLAGS+= CWARNFLAGS.clang=-Wno-error=missing-noreturn
15 15
16CONFLICTS= pbulk<0.39 16CONFLICTS= pbulk<0.39

cvs diff -r1.18 -r1.19 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c (expand / switch to unified diff)

--- pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c 2023/02/10 23:14:32 1.18
+++ pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c 2023/02/12 04:12:54 1.19
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: jobs.c,v 1.18 2023/02/10 23:14:32 joerg Exp $ */ 1/* $NetBSD: jobs.c,v 1.19 2023/02/12 04:12:54 joerg Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2007, 2009, 2011 Joerg Sonnenberger <joerg@NetBSD.org>. 4 * Copyright (c) 2007, 2009, 2011 Joerg Sonnenberger <joerg@NetBSD.org>.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * This code was developed as part of Google's Summer of Code 2007 program. 7 * This code was developed as part of Google's Summer of Code 2007 program.
8 * 8 *
9 * Redistribution and use in source and binary forms, with or without 9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions 10 * modification, are permitted provided that the following conditions
11 * are met: 11 * are met:
12 * 12 *
13 * 1. Redistributions of source code must retain the above copyright 13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer. 14 * notice, this list of conditions and the following disclaimer.
@@ -154,41 +154,43 @@ compute_tree_depth_rec(struct build_job  @@ -154,41 +154,43 @@ compute_tree_depth_rec(struct build_job
154{ 154{
155 struct dependency_list *dep_iter; 155 struct dependency_list *dep_iter;
156 struct build_job *job_iter; 156 struct build_job *job_iter;
157 157
158 if (job == root && root->pkg_depth != 0) { 158 if (job == root && root->pkg_depth != 0) {
159 fprintf(stderr, "Cyclic dependency for package:\n%s\n", job->pkgname); 159 fprintf(stderr, "Cyclic dependency for package:\n%s\n", job->pkgname);
160 return -1; 160 return -1;
161 } 161 }
162 162
163 if (seen[job - jobs]) 163 if (seen[job - jobs])
164 return 0; 164 return 0;
165 seen[job - jobs] = 1; 165 seen[job - jobs] = 1;
166 ++root->pkg_depth; 166 ++root->pkg_depth;
 167 root->pkg_weighted_depth += job->pkg_weight;
167 SLIST_FOREACH(dep_iter, &job->depending_pkgs, depends_link) { 168 SLIST_FOREACH(dep_iter, &job->depending_pkgs, depends_link) {
168 if (compute_tree_depth_rec(dep_iter->dependency, root, seen)) { 169 if (compute_tree_depth_rec(dep_iter->dependency, root, seen)) {
169 fprintf(stderr, "%s\n", job->pkgname); 170 fprintf(stderr, "%s\n", job->pkgname);
170 return -1; 171 return -1;
171 } 172 }
172 } 173 }
173 return 0; 174 return 0;
174} 175}
175 176
176static void 177static void
177compute_tree_depth(struct build_job *job, char *seen) 178compute_tree_depth(struct build_job *job, char *seen)
178{ 179{
179 memset(seen, 0, len_jobs); 180 memset(seen, 0, len_jobs);
180 181
181 job->pkg_depth = 0; 182 job->pkg_depth = 0;
 183 job->pkg_weighted_depth = 0;
182 if (compute_tree_depth_rec(job, job, seen)) 184 if (compute_tree_depth_rec(job, job, seen))
183 exit(1); 185 exit(1);
184} 186}
185 187
186void 188void
187init_jobs(const char *scan_output, const char *success_file, const char *error_file) 189init_jobs(const char *scan_output, const char *success_file, const char *error_file)
188{ 190{
189 char *input, *seen; 191 char *input, *seen;
190 const char *input_iter; 192 const char *input_iter;
191 int fd; 193 int fd;
192 size_t i; 194 size_t i;
193 195
194 TAILQ_INIT(&buildable_jobs); 196 TAILQ_INIT(&buildable_jobs);
@@ -271,79 +273,104 @@ mark_initial_state(int fd, enum job_stat @@ -271,79 +273,104 @@ mark_initial_state(int fd, enum job_stat
271 errx(1, "Package from %s build log doesn't exist: %.*s", type, (int)len, input_iter); 273 errx(1, "Package from %s build log doesn't exist: %.*s", type, (int)len, input_iter);
272 274
273 process_job(iter, state, 0); 275 process_job(iter, state, 0);
274 276
275 input_iter = strchr(input_iter, '\n'); 277 input_iter = strchr(input_iter, '\n');
276 if (input_iter == NULL) 278 if (input_iter == NULL)
277 errx(1, "Invalid input"); 279 errx(1, "Invalid input");
278 ++input_iter; 280 ++input_iter;
279 } 281 }
280 free(input); 282 free(input);
281} 283}
282 284
283static void 285static void
 286parse_weight(struct build_job *job)
 287{
 288 const char *line;
 289 char *eos;
 290 long long value;
 291
 292 line = find_content(job, "PBULK_WEIGHT=");
 293 if (line == NULL) {
 294 job->pkg_weight = 100;
 295 return;
 296 }
 297 errno = 0;
 298 value = strtoll(line, &eos, 10);
 299 if (errno || line == eos || *eos != '\n')
 300 errx(1, "Invalid PBULK_WEIGHT for package %s", job->pkgname);
 301 if (value < 0 && value <= LLONG_MIN / len_jobs)
 302 errx(1, "PBULK_WEIGHT too small for package %s", job->pkgname);
 303 if (value > 0 && value >= LLONG_MAX / len_jobs)
 304 errx(1, "PBULK_WEIGHT too large for package %s", job->pkgname);
 305 job->pkg_weight = value;
 306}
 307
 308static void
284mark_initial(void) 309mark_initial(void)
285{ 310{
286 const char *line; 311 const char *line;
287 size_t i; 312 size_t i;
288 313
289 for (i = 0; i < len_jobs; ++i) { 314 for (i = 0; i < len_jobs; ++i) {
290 if ((line = find_content(&jobs[i], "PKG_SKIP_REASON=")) == NULL) 315 if ((line = find_content(&jobs[i], "PKG_SKIP_REASON=")) == NULL)
291 errx(1, "Invalid scan output"); 316 errx(1, "Invalid scan output");
292 if (*line != '\n') { 317 if (*line != '\n') {
293 process_job(&jobs[i], JOB_PREFAILED, 0); 318 process_job(&jobs[i], JOB_PREFAILED, 0);
294 continue; 319 continue;
295 } 320 }
296 if ((line = find_content(&jobs[i], "PKG_FAIL_REASON=")) == NULL) 321 if ((line = find_content(&jobs[i], "PKG_FAIL_REASON=")) == NULL)
297 errx(1, "Invalid scan output"); 322 errx(1, "Invalid scan output");
298 if (*line != '\n') { 323 if (*line != '\n') {
299 process_job(&jobs[i], JOB_PREFAILED, 0); 324 process_job(&jobs[i], JOB_PREFAILED, 0);
300 continue; 325 continue;
301 } 326 }
302 }  327 }
303 328
304 mark_initial_state(log_success, JOB_DONE, "successful"); 329 mark_initial_state(log_success, JOB_DONE, "successful");
305 mark_initial_state(log_failed, JOB_FAILED, "failing"); 330 mark_initial_state(log_failed, JOB_FAILED, "failing");
306 331
307 if (verbosity >= 1) 332 if (verbosity >= 1)
308 printf("Initialisation complete.\n"); 333 printf("Initialisation complete.\n");
309 post_initial = 1; 334 post_initial = 1;
310} 335}
311 336
312static void 337static void
313add_to_build_list(struct build_job *job) 338add_to_build_list(struct build_job *job)
314{ 339{
315 struct build_job *iter; 340 struct build_job *iter;
316 341
317 TAILQ_FOREACH(iter, &buildable_jobs, build_link) { 342 TAILQ_FOREACH(iter, &buildable_jobs, build_link) {
318 if (iter->pkg_depth < job->pkg_depth) { 343 if (iter->pkg_weighted_depth < job->pkg_weighted_depth) {
319 TAILQ_INSERT_BEFORE(iter, job, build_link); 344 TAILQ_INSERT_BEFORE(iter, job, build_link);
320 return; 345 return;
321 } 346 }
322 } 347 }
323 TAILQ_INSERT_TAIL(&buildable_jobs, job, build_link); 348 TAILQ_INSERT_TAIL(&buildable_jobs, job, build_link);
324} 349}
325 350
326static void 351static void
327build_tree(void) 352build_tree(void)
328{ 353{
329 size_t i, len; 354 size_t i, len;
330 struct build_job *iter, *job; 355 struct build_job *iter, *job;
331 struct dependency_list *dep; 356 struct dependency_list *dep;
332 const char *depends; 357 const char *depends;
333 358
334 for (i = 0; i < len_jobs; ++i) { 359 for (i = 0; i < len_jobs; ++i) {
335 job = &jobs[i]; 360 job = &jobs[i];
336 361
 362 parse_weight(job);
 363
337 if ((depends = find_content(job, "DEPENDS=")) == NULL) 364 if ((depends = find_content(job, "DEPENDS=")) == NULL)
338 continue; 365 continue;
339 366
340 depends += strspn(depends, " \t"); 367 depends += strspn(depends, " \t");
341 368
342 while ((len = strcspn(depends, " \t\n")) != 0) { 369 while ((len = strcspn(depends, " \t\n")) != 0) {
343 SLIST_FOREACH(iter, get_hash_chain(depends, len), hash_link) { 370 SLIST_FOREACH(iter, get_hash_chain(depends, len), hash_link) {
344 if (iter->pkgname_len == len && 371 if (iter->pkgname_len == len &&
345 strncmp(iter->pkgname, depends, len) == 0) 372 strncmp(iter->pkgname, depends, len) == 0)
346 break; 373 break;
347 } 374 }
348 if (iter == NULL) 375 if (iter == NULL)
349 errx(1, "Dependency `%.*s' doesn't exist", (int)strcspn(depends, " \t\n"), depends); 376 errx(1, "Dependency `%.*s' doesn't exist", (int)strcspn(depends, " \t\n"), depends);
@@ -520,28 +547,28 @@ finish_build(const char *report_file) @@ -520,28 +547,28 @@ finish_build(const char *report_file)
520 break; 547 break;
521 case JOB_INDIRECT_PREFAILED: 548 case JOB_INDIRECT_PREFAILED:
522 status = "indirect-prefailed"; 549 status = "indirect-prefailed";
523 break; 550 break;
524 case JOB_OPEN: 551 case JOB_OPEN:
525 status = "open"; 552 status = "open";
526 break; 553 break;
527 case JOB_IN_PROCESSING: 554 case JOB_IN_PROCESSING:
528 status = "in_processing"; 555 status = "in_processing";
529 break; 556 break;
530 default: 557 default:
531 errx(1, "internal error"); 558 errx(1, "internal error");
532 } 559 }
533 fprintf(report, "%s|%s||%d\n", jobs[i].pkgname, status, 560 fprintf(report, "%s|%s||%d|%lld\n", jobs[i].pkgname, status,
534 jobs[i].pkg_depth); 561 jobs[i].pkg_depth, jobs[i].pkg_weighted_depth);
535 } 562 }
536} 563}
537 564
538#define HASH_SIZE 4096 565#define HASH_SIZE 4096
539 566
540static size_t 567static size_t
541hash_item(const char *s, size_t len) 568hash_item(const char *s, size_t len)
542{ 569{
543 return djb_hash2(s, s + len) % HASH_SIZE; 570 return djb_hash2(s, s + len) % HASH_SIZE;
544} 571}
545 572
546static struct buildhash hash_table[HASH_SIZE]; 573static struct buildhash hash_table[HASH_SIZE];
547 574

cvs diff -r1.8 -r1.9 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h (expand / switch to unified diff)

--- pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h 2023/02/10 23:14:32 1.8
+++ pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h 2023/02/12 04:12:54 1.9
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: pbuild.h,v 1.8 2023/02/10 23:14:32 joerg Exp $ */ 1/* $NetBSD: pbuild.h,v 1.9 2023/02/12 04:12:54 joerg Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>. 4 * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * This code was developed as part of Google's Summer of Code 2007 program. 7 * This code was developed as part of Google's Summer of Code 2007 program.
8 * 8 *
9 * Redistribution and use in source and binary forms, with or without 9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions 10 * modification, are permitted provided that the following conditions
11 * are met: 11 * are met:
12 * 12 *
13 * 1. Redistributions of source code must retain the above copyright 13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer. 14 * notice, this list of conditions and the following disclaimer.
@@ -86,26 +86,28 @@ struct build_job { @@ -86,26 +86,28 @@ struct build_job {
86 86
87 /** 87 /**
88 * Pointers into the output from pbulk-resolve. The lines 88 * Pointers into the output from pbulk-resolve. The lines
89 * between these two pointers describe additional properties 89 * between these two pointers describe additional properties
90 * of the job, such as the PKGPATH in which to build the 90 * of the job, such as the PKGPATH in which to build the
91 * package. The information can be accessed with the 91 * package. The information can be accessed with the
92 * find_content function. 92 * find_content function.
93 */ 93 */
94 const char *begin; 94 const char *begin;
95 const char *end; 95 const char *end;
96 96
97 enum job_state state; 97 enum job_state state;
98 int pkg_depth; 98 int pkg_depth;
 99 long long pkg_weighted_depth;
 100 long long pkg_weight;
99 101
100 /** 102 /**
101 * The number of direct dependencies that must be built before 103 * The number of direct dependencies that must be built before
102 * this package can be tried. 104 * this package can be tried.
103 */ 105 */
104 size_t open_depends; 106 size_t open_depends;
105 107
106 /** The packages that depend on this package. */ 108 /** The packages that depend on this package. */
107 SLIST_HEAD(, dependency_list) depending_pkgs; 109 SLIST_HEAD(, dependency_list) depending_pkgs;
108 110
109 TAILQ_ENTRY(build_job) build_link; 111 TAILQ_ENTRY(build_job) build_link;
110 SLIST_ENTRY(build_job) hash_link; 112 SLIST_ENTRY(build_job) hash_link;
111}; 113};