Sun Feb 12 21:17:24 2023 UTC ()
pbulk-0.57: switch to a binary heap for the build queue


(joerg)
diff -r1.27 -r1.28 pkgsrc/pkgtools/pbulk-base/Makefile
diff -r1.19 -r1.20 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c
diff -r1.9 -r1.10 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h

cvs diff -r1.27 -r1.28 pkgsrc/pkgtools/pbulk-base/Makefile (expand / switch to unified diff)

--- pkgsrc/pkgtools/pbulk-base/Makefile 2023/02/12 04:12:54 1.27
+++ pkgsrc/pkgtools/pbulk-base/Makefile 2023/02/12 21:17:24 1.28
@@ -1,16 +1,16 @@ @@ -1,16 +1,16 @@
1# $NetBSD: Makefile,v 1.27 2023/02/12 04:12:54 joerg Exp $ 1# $NetBSD: Makefile,v 1.28 2023/02/12 21:17:24 joerg Exp $
2 2
3DISTNAME= pbulk-base-0.56 3DISTNAME= pbulk-base-0.57
4COMMENT= Core components of the modular bulk build framework 4COMMENT= Core components of the modular bulk build framework
5 5
6.include "../../pkgtools/pbulk/Makefile.common" 6.include "../../pkgtools/pbulk/Makefile.common"
7 7
8USE_FEATURES= nbcompat 8USE_FEATURES= nbcompat
9USE_TOOLS+= nroff 9USE_TOOLS+= nroff
10 10
11INSTALLATION_DIRS= bin ${PKGMANDIR}/cat1 ${PKGMANDIR}/man1 11INSTALLATION_DIRS= bin ${PKGMANDIR}/cat1 ${PKGMANDIR}/man1
12USE_BSD_MAKEFILE= yes 12USE_BSD_MAKEFILE= yes
13 13
14MAKE_FLAGS+= CWARNFLAGS.clang=-Wno-error=missing-noreturn 14MAKE_FLAGS+= CWARNFLAGS.clang=-Wno-error=missing-noreturn
15 15
16CONFLICTS= pbulk<0.39 16CONFLICTS= pbulk<0.39

cvs diff -r1.19 -r1.20 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c (expand / switch to unified diff)

--- pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c 2023/02/12 04:12:54 1.19
+++ pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c 2023/02/12 21:17:24 1.20
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: jobs.c,v 1.19 2023/02/12 04:12:54 joerg Exp $ */ 1/* $NetBSD: jobs.c,v 1.20 2023/02/12 21:17:24 joerg Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2007, 2009, 2011 Joerg Sonnenberger <joerg@NetBSD.org>. 4 * Copyright (c) 2007, 2009, 2011 Joerg Sonnenberger <joerg@NetBSD.org>.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * This code was developed as part of Google's Summer of Code 2007 program. 7 * This code was developed as part of Google's Summer of Code 2007 program.
8 * 8 *
9 * Redistribution and use in source and binary forms, with or without 9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions 10 * modification, are permitted provided that the following conditions
11 * are met: 11 * are met:
12 * 12 *
13 * 1. Redistributions of source code must retain the above copyright 13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer. 14 * notice, this list of conditions and the following disclaimer.
@@ -54,27 +54,28 @@ static int log_failed; @@ -54,27 +54,28 @@ static int log_failed;
54 54
55SLIST_HEAD(buildhash, build_job); 55SLIST_HEAD(buildhash, build_job);
56 56
57static void build_tree(void); 57static void build_tree(void);
58static void mark_initial(void); 58static void mark_initial(void);
59static struct buildhash *get_hash_chain(const char *, size_t); 59static struct buildhash *get_hash_chain(const char *, size_t);
60static void hash_entries(void); 60static void hash_entries(void);
61static void add_to_build_list(struct build_job *); 61static void add_to_build_list(struct build_job *);
62 62
63static struct build_job *jobs; 63static struct build_job *jobs;
64static size_t allocated_jobs, len_jobs; 64static size_t allocated_jobs, len_jobs;
65static char *scan_output_content; 65static char *scan_output_content;
66 66
67static TAILQ_HEAD(, build_job) buildable_jobs; 67static struct build_job **buildable_jobs;
 68static size_t buildable_jobs_len;
68 69
69#if defined(__GNUC__) && __GNUC__ >= 2 70#if defined(__GNUC__) && __GNUC__ >= 2
70__attribute__((__format__(__printf__, 1, 2))) 71__attribute__((__format__(__printf__, 1, 2)))
71#endif 72#endif
72static void 73static void
73ts_printf(const char *fmt, ...) 74ts_printf(const char *fmt, ...)
74{ 75{
75 struct tm *tm; 76 struct tm *tm;
76 time_t now; 77 time_t now;
77 va_list ap; 78 va_list ap;
78 char buf[512]; 79 char buf[512];
79 struct build_stat st; 80 struct build_stat st;
80 81
@@ -183,28 +184,26 @@ compute_tree_depth(struct build_job *job @@ -183,28 +184,26 @@ compute_tree_depth(struct build_job *job
183 job->pkg_weighted_depth = 0; 184 job->pkg_weighted_depth = 0;
184 if (compute_tree_depth_rec(job, job, seen)) 185 if (compute_tree_depth_rec(job, job, seen))
185 exit(1); 186 exit(1);
186} 187}
187 188
188void 189void
189init_jobs(const char *scan_output, const char *success_file, const char *error_file) 190init_jobs(const char *scan_output, const char *success_file, const char *error_file)
190{ 191{
191 char *input, *seen; 192 char *input, *seen;
192 const char *input_iter; 193 const char *input_iter;
193 int fd; 194 int fd;
194 size_t i; 195 size_t i;
195 196
196 TAILQ_INIT(&buildable_jobs); 
197 
198 if ((fd = open(scan_output, O_RDONLY, 0)) == -1) 197 if ((fd = open(scan_output, O_RDONLY, 0)) == -1)
199 err(1, "Cannot open input"); 198 err(1, "Cannot open input");
200 199
201 log_success = open(success_file, O_RDWR | O_CREAT | O_APPEND, 0666); 200 log_success = open(success_file, O_RDWR | O_CREAT | O_APPEND, 0666);
202 if (log_success == -1) 201 if (log_success == -1)
203 err(1, "Cannot open log file for successful builds"); 202 err(1, "Cannot open log file for successful builds");
204 203
205 log_failed = open(error_file, O_RDWR | O_CREAT | O_APPEND, 0666); 204 log_failed = open(error_file, O_RDWR | O_CREAT | O_APPEND, 0666);
206 if (log_failed == -1) 205 if (log_failed == -1)
207 err(1, "Cannot open log file for failed builds"); 206 err(1, "Cannot open log file for failed builds");
208 207
209 input = read_from_file(fd); 208 input = read_from_file(fd);
210 (void)close(fd); 209 (void)close(fd);
@@ -225,26 +224,29 @@ init_jobs(const char *scan_output, const @@ -225,26 +224,29 @@ init_jobs(const char *scan_output, const
225 errx(1, "Invalid input"); 224 errx(1, "Invalid input");
226 input_iter = jobs[len_jobs].end; 225 input_iter = jobs[len_jobs].end;
227 ++len_jobs; 226 ++len_jobs;
228 if (len_jobs == allocated_jobs) { 227 if (len_jobs == allocated_jobs) {
229 allocated_jobs *= 2; 228 allocated_jobs *= 2;
230 jobs = xrealloc(jobs, allocated_jobs * sizeof(*jobs)); 229 jobs = xrealloc(jobs, allocated_jobs * sizeof(*jobs));
231 } 230 }
232 } 231 }
233 232
234 if (*input_iter != '\0') 233 if (*input_iter != '\0')
235 errx(1, "Invalid input"); 234 errx(1, "Invalid input");
236 scan_output_content = input; 235 scan_output_content = input;
237 236
 237 buildable_jobs = xmalloc(len_jobs * sizeof(*buildable_jobs));
 238 buildable_jobs_len = 0;
 239
238 hash_entries(); 240 hash_entries();
239 build_tree(); 241 build_tree();
240 242
241 seen = xmalloc(len_jobs); 243 seen = xmalloc(len_jobs);
242 for (i = 0; i < len_jobs; ++i) 244 for (i = 0; i < len_jobs; ++i)
243 compute_tree_depth(&jobs[i], seen); 245 compute_tree_depth(&jobs[i], seen);
244 free(seen); 246 free(seen);
245 247
246 mark_initial(); 248 mark_initial();
247 249
248 for (i = 0; i < len_jobs; ++i) { 250 for (i = 0; i < len_jobs; ++i) {
249 if (jobs[i].state == JOB_INIT) 251 if (jobs[i].state == JOB_INIT)
250 process_job(&jobs[i], JOB_OPEN, 0); 252 process_job(&jobs[i], JOB_OPEN, 0);
@@ -325,37 +327,51 @@ mark_initial(void) @@ -325,37 +327,51 @@ mark_initial(void)
325 continue; 327 continue;
326 } 328 }
327 } 329 }
328 330
329 mark_initial_state(log_success, JOB_DONE, "successful"); 331 mark_initial_state(log_success, JOB_DONE, "successful");
330 mark_initial_state(log_failed, JOB_FAILED, "failing"); 332 mark_initial_state(log_failed, JOB_FAILED, "failing");
331 333
332 if (verbosity >= 1) 334 if (verbosity >= 1)
333 printf("Initialisation complete.\n"); 335 printf("Initialisation complete.\n");
334 post_initial = 1; 336 post_initial = 1;
335} 337}
336 338
337static void 339static void
 340swap_buildable_entries(size_t i, size_t j)
 341{
 342 struct build_job *job1, *job2;
 343
 344 job1 = buildable_jobs[i];
 345 job2 = buildable_jobs[j];
 346 job1->buildable_index = j;
 347 job2->buildable_index = i;
 348 buildable_jobs[i] = job2;
 349 buildable_jobs[j] = job1;
 350}
 351
 352static void
338add_to_build_list(struct build_job *job) 353add_to_build_list(struct build_job *job)
339{ 354{
340 struct build_job *iter; 355 size_t idx;
341 356
342 TAILQ_FOREACH(iter, &buildable_jobs, build_link) { 357 job->buildable_index = buildable_jobs_len++;
343 if (iter->pkg_weighted_depth < job->pkg_weighted_depth) { 358 buildable_jobs[job->buildable_index] = job;
344 TAILQ_INSERT_BEFORE(iter, job, build_link); 359 while ((idx = job->buildable_index)) {
345 return; 360 if (buildable_jobs[(idx - 1) / 2]->pkg_weighted_depth >=
346 } 361 job->pkg_weighted_depth)
 362 break;
 363 swap_buildable_entries(idx, (idx - 1) / 2);
347 } 364 }
348 TAILQ_INSERT_TAIL(&buildable_jobs, job, build_link); 
349} 365}
350 366
351static void 367static void
352build_tree(void) 368build_tree(void)
353{ 369{
354 size_t i, len; 370 size_t i, len;
355 struct build_job *iter, *job; 371 struct build_job *iter, *job;
356 struct dependency_list *dep; 372 struct dependency_list *dep;
357 const char *depends; 373 const char *depends;
358 374
359 for (i = 0; i < len_jobs; ++i) { 375 for (i = 0; i < len_jobs; ++i) {
360 job = &jobs[i]; 376 job = &jobs[i];
361 377
@@ -379,39 +395,76 @@ build_tree(void) @@ -379,39 +395,76 @@ build_tree(void)
379 dep->dependency = job; 395 dep->dependency = job;
380 SLIST_INSERT_HEAD(&iter->depending_pkgs, dep, depends_link); 396 SLIST_INSERT_HEAD(&iter->depending_pkgs, dep, depends_link);
381 ++job->open_depends; 397 ++job->open_depends;
382 398
383 depends += len; 399 depends += len;
384 depends += strspn(depends, " \t"); 400 depends += strspn(depends, " \t");
385 } 401 }
386 } 402 }
387} 403}
388 404
389int 405int
390has_job(void) 406has_job(void)
391{ 407{
392 return !TAILQ_EMPTY(&buildable_jobs); 408 return buildable_jobs_len != 0;
 409}
 410
 411static void
 412buildable_jobs_remove_head(void)
 413{
 414 size_t idx, idx2;
 415
 416 --buildable_jobs_len;
 417 if (buildable_jobs_len == 0)
 418 return;
 419
 420 /* Move last element to head */
 421 buildable_jobs[0] = buildable_jobs[buildable_jobs_len];
 422 buildable_jobs[0]->buildable_index = 0;
 423
 424 idx = 0;
 425 for (;;) {
 426 /* Find biggest child. */
 427 idx2 = idx;
 428 if (idx * 2 + 1 < buildable_jobs_len &&
 429 buildable_jobs[2 * idx + 1]->pkg_weighted_depth >
 430 buildable_jobs[idx2]->pkg_weighted_depth) {
 431 idx2 = 2 * idx + 1;
 432 }
 433 if (idx * 2 + 2 < buildable_jobs_len &&
 434 buildable_jobs[2 * idx + 2]->pkg_weighted_depth >
 435 buildable_jobs[idx2]->pkg_weighted_depth) {
 436 idx2 = 2 * idx + 2;
 437 }
 438 if (idx == idx2)
 439 break;
 440 swap_buildable_entries(idx, idx2);
 441 idx = idx2;
 442 }
393} 443}
394 444
395struct build_job * 445struct build_job *
396get_job(void) 446get_job(void)
397{ 447{
398 struct build_job *job; 448 struct build_job *job;
399 449
400 if ((job = TAILQ_FIRST(&buildable_jobs)) != NULL) { 450 if (buildable_jobs_len == 0)
401 TAILQ_REMOVE(&buildable_jobs, job, build_link); 451 return NULL;
 452
 453 job = buildable_jobs[0];
 454 process_job(job, JOB_IN_PROCESSING, 0);
 455
 456 buildable_jobs_remove_head();
402 457
403 process_job(job, JOB_IN_PROCESSING, 0); 
404 } 
405 return job; 458 return job;
406} 459}
407 460
408static void 461static void
409recursive_mark_broken(struct build_job *job, enum job_state state) 462recursive_mark_broken(struct build_job *job, enum job_state state)
410{ 463{
411 struct dependency_list *iter; 464 struct dependency_list *iter;
412 465
413 SLIST_FOREACH(iter, &job->depending_pkgs, depends_link) { 466 SLIST_FOREACH(iter, &job->depending_pkgs, depends_link) {
414 if (iter->dependency->state == JOB_OPEN || 467 if (iter->dependency->state == JOB_OPEN ||
415 iter->dependency->state == JOB_INIT) 468 iter->dependency->state == JOB_INIT)
416 process_job(iter->dependency, state, 0); 469 process_job(iter->dependency, state, 0);
417 }  470 }

cvs diff -r1.9 -r1.10 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h (expand / switch to unified diff)

--- pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h 2023/02/12 04:12:54 1.9
+++ pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h 2023/02/12 21:17:24 1.10
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: pbuild.h,v 1.9 2023/02/12 04:12:54 joerg Exp $ */ 1/* $NetBSD: pbuild.h,v 1.10 2023/02/12 21:17:24 joerg Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>. 4 * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * This code was developed as part of Google's Summer of Code 2007 program. 7 * This code was developed as part of Google's Summer of Code 2007 program.
8 * 8 *
9 * Redistribution and use in source and binary forms, with or without 9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions 10 * modification, are permitted provided that the following conditions
11 * are met: 11 * are met:
12 * 12 *
13 * 1. Redistributions of source code must retain the above copyright 13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer. 14 * notice, this list of conditions and the following disclaimer.
@@ -98,27 +98,27 @@ struct build_job { @@ -98,27 +98,27 @@ struct build_job {
98 int pkg_depth; 98 int pkg_depth;
99 long long pkg_weighted_depth; 99 long long pkg_weighted_depth;
100 long long pkg_weight; 100 long long pkg_weight;
101 101
102 /** 102 /**
103 * The number of direct dependencies that must be built before 103 * The number of direct dependencies that must be built before
104 * this package can be tried. 104 * this package can be tried.
105 */ 105 */
106 size_t open_depends; 106 size_t open_depends;
107 107
108 /** The packages that depend on this package. */ 108 /** The packages that depend on this package. */
109 SLIST_HEAD(, dependency_list) depending_pkgs; 109 SLIST_HEAD(, dependency_list) depending_pkgs;
110 110
111 TAILQ_ENTRY(build_job) build_link; 111 size_t buildable_index;
112 SLIST_ENTRY(build_job) hash_link; 112 SLIST_ENTRY(build_job) hash_link;
113}; 113};
114 114
115extern int verbosity; 115extern int verbosity;
116 116
117void init_jobs(const char *, const char *, const char *); 117void init_jobs(const char *, const char *, const char *);
118int has_job(void); 118int has_job(void);
119struct build_job *get_job(void); 119struct build_job *get_job(void);
120void process_job(struct build_job *, enum job_state, int); 120void process_job(struct build_job *, enum job_state, int);
121int build_package(const char *, size_t); 121int build_package(const char *, size_t);
122void finish_build(const char *); 122void finish_build(const char *);
123void build_stats(struct build_stat *); 123void build_stats(struct build_stat *);
124 124