cbaines pushed a commit to branch master
in repository data-service.
commit 50127de12a56177f7e953a8d8a01cea8f779adb7
Author: Christopher Baines <[email protected]>
AuthorDate: Sat Dec 21 08:39:36 2024 +0000
Parallelise inserting derivation counts
---
guix-data-service/jobs/load-new-guix-revision.scm | 59 +++++++++++++----------
1 file changed, 33 insertions(+), 26 deletions(-)
diff --git a/guix-data-service/jobs/load-new-guix-revision.scm
b/guix-data-service/jobs/load-new-guix-revision.scm
index 83a43a0..ad3f1ed 100644
--- a/guix-data-service/jobs/load-new-guix-revision.scm
+++ b/guix-data-service/jobs/load-new-guix-revision.scm
@@ -2882,38 +2882,45 @@ SKIP LOCKED")
(when result
(fibers-parallel
- (with-postgresql-connection
- (simple-format #f "post load-new-guix-revision ~A counts" id)
- (lambda (conn)
- (let ((revision-id
- (guix-revision-id-for-job conn id)))
- (with-time-logging
- (simple-format
- #f "inserting
guix_revision_package_derivation_distribution_counts for revision ~A"
- revision-id)
- (let ((system-ids-and-targets
- (exec-query
- conn
- "
+ (let ((revision-id
+ system-ids-and-targets
+ (with-postgresql-connection
+ (simple-format #f "post load-new-guix-revision ~A counts" id)
+ (lambda (conn)
+ (let ((revision-id
+ (guix-revision-id-for-job conn id)))
+ (values
+ revision-id
+ (exec-query
+ conn
+ "
SELECT DISTINCT system_id, target
FROM package_derivations
INNER JOIN guix_revision_package_derivations
ON package_derivations.id =
guix_revision_package_derivations.package_derivation_id
WHERE revision_id = $1"
- (list revision-id))))
+ (list revision-id))))))))
- (with-postgresql-transaction
- conn
- (lambda (conn)
- (for-each
- (match-lambda
- ((system-id target)
-
(insert-guix-revision-package-derivation-distribution-counts
- conn
- revision-id
- system-id
- target)))
- system-ids-and-targets))))))))
+ (with-time-logging
+ (simple-format
+ #f "inserting guix_revision_package_derivation_distribution_counts
for revision ~A"
+ revision-id)
+ (fibers-batch-for-each
+ (match-lambda
+ ((system-id target)
+ (with-postgresql-connection
+ (simple-format #f "post load-new-guix-revision ~A counts" id)
+ (lambda (conn)
+ (with-postgresql-transaction
+ conn
+ (lambda (conn)
+ (insert-guix-revision-package-derivation-distribution-counts
+ conn
+ revision-id
+ system-id
+ target)))))))
+ parallelism
+ system-ids-and-targets)))
(with-postgresql-connection
(simple-format #f "post load-new-guix-revision ~A" id)
