treewide: drop -l$NIX_BUILD_CORES

Passing `-l$NIX_BUILD_CORES` improperly limits the overall system load. For a build machine which is configured to run `$B` builds where each build gets `total cores / B` cores (`$C`), passing `-l $C` to make will improperly limit the load to `$C` instead of `$B * $C`. This effect becomes quite pronounced on machines with 80 cores, with 40 simultaneous builds and a cores limit of 2. On a machine with this configuration, Nix will run 40 builds and make will limit the overall system load to approximately 2. A build machine with this many cores can happily run with a load approaching 80. A non-solution is to oversubscribe the machine, by picking a larger `$C`. However, there is no way to divide the number of cores in a way which fairly subdivides the available cores when `$B` is greater than 1. There has been exploration of passing a jobserver in to the sandbox, or sharing a jobserver between all the builds. This is one option, but relatively complicated and only supports make. Lots of other software uses its own implementation of `-j` and doesn't support either `-l` or the Make jobserver. For the case of an interactive user machine, the user should limit overall system load using `$B`, `$C`, and optionally systemd's cpu/network/io limiting features. Making this change should significantly improve the utilization of our build farm, and improve the throughput of Hydra.
NixOS · Sep 22, 2022 · c2b898d · c2b898d
1 parent a3f8caa
commit c2b898d
Show file tree

Hide file tree

Showing 15 changed files with 21 additions and 21 deletions.
diff --git a/pkgs/applications/emulators/np2kai/default.nix b/pkgs/applications/emulators/np2kai/default.nix
@@ -124,7 +124,7 @@ stdenv.mkDerivation rec {
 
   configurePhase = ''
     export GIT_VERSION=${builtins.substring 0 7 src.rev}
-    buildFlags="$buildFlags ''${enableParallelBuilding:+-j$NIX_BUILD_CORES -l$NIX_BUILD_CORES}"
+    buildFlags="$buildFlags ''${enableParallelBuilding:+-j$NIX_BUILD_CORES"
   '' + optionalString enableX11 ''
     cd x11
     substituteInPlace Makefile.am \

diff --git a/pkgs/applications/networking/browsers/chromium/common.nix b/pkgs/applications/networking/browsers/chromium/common.nix
@@ -325,7 +325,7 @@ let
 
     buildPhase = let
       buildCommand = target: ''
-        ninja -C "${buildPath}" -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES "${target}"
+        ninja -C "${buildPath}" -j$NIX_BUILD_CORES "${target}"
         (
           source chrome/installer/linux/common/installer.include
           PACKAGE=$packageName

diff --git a/pkgs/applications/version-management/git-and-tools/git/default.nix b/pkgs/applications/version-management/git-and-tools/git/default.nix
@@ -252,7 +252,7 @@ stdenv.mkDerivation (finalAttrs: {
       '')
 
    + lib.optionalString withManual ''# Install man pages
-       make -j $NIX_BUILD_CORES -l $NIX_BUILD_CORES PERL_PATH="${buildPackages.perl}/bin/perl" cmd-list.made install install-html \
+       make -j $NIX_BUILD_CORES PERL_PATH="${buildPackages.perl}/bin/perl" cmd-list.made install install-html \
          -C Documentation ''
 
    + (if guiSupport then ''

diff --git a/pkgs/development/embedded/blackmagic/helper.sh b/pkgs/development/embedded/blackmagic/helper.sh
@@ -11,7 +11,7 @@ out=${out:-/tmp}
 ################################################################################
 export CFLAGS=$NIX_CFLAGS_COMPILE
 export MAKEFLAGS="\
-  ${enableParallelBuilding:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}"
+  ${enableParallelBuilding:+-j${NIX_BUILD_CORES}}"
 
 ################################################################################
 PRODUCTS="blackmagic.bin blackmagic.hex blackmagic_dfu.bin blackmagic_dfu.hex"

diff --git a/pkgs/development/libraries/glibc/default.nix b/pkgs/development/libraries/glibc/default.nix
@@ -81,7 +81,7 @@ callPackage ./common.nix { inherit stdenv; } {
 
     postInstall = (if stdenv.hostPlatform == stdenv.buildPlatform then ''
       echo SUPPORTED-LOCALES=C.UTF-8/UTF-8 > ../glibc-2*/localedata/SUPPORTED
-      make -j''${NIX_BUILD_CORES:-1} -l''${NIX_BUILD_CORES:-1} localedata/install-locales
+      make -j''${NIX_BUILD_CORES:-1} localedata/install-locales
     '' else lib.optionalString stdenv.buildPlatform.isLinux ''
       # This is based on http://www.linuxfromscratch.org/lfs/view/development/chapter06/glibc.html
       # Instead of using their patch to build a build-native localedef,

diff --git a/pkgs/development/libraries/kerberos/krb5.nix b/pkgs/development/libraries/kerberos/krb5.nix
@@ -55,7 +55,7 @@ stdenv.mkDerivation rec {
   buildPhase = lib.optionalString libOnly ''
     runHook preBuild
 
-    MAKE="make -j $NIX_BUILD_CORES -l $NIX_BUILD_CORES"
+    MAKE="make -j $NIX_BUILD_CORES"
     for folder in $libFolders; do
       $MAKE -C $folder
     done

diff --git a/pkgs/development/python-modules/pybind11/default.nix b/pkgs/development/python-modules/pybind11/default.nix
@@ -41,7 +41,7 @@ buildPythonPackage rec {
 
   postBuild = ''
     # build tests
-    make -j $NIX_BUILD_CORES -l $NIX_BUILD_CORES
+    make -j $NIX_BUILD_CORES
   '';
 
   postInstall = ''

diff --git a/pkgs/development/tools/build-managers/ninja/setup-hook.sh b/pkgs/development/tools/build-managers/ninja/setup-hook.sh
@@ -9,7 +9,7 @@ ninjaBuildPhase() {
     fi
 
     local flagsArray=(
-        -j$buildCores -l$NIX_BUILD_CORES
+        -j$buildCores
         $ninjaFlags "${ninjaFlagsArray[@]}"
     )
 
@@ -61,7 +61,7 @@ ninjaCheckPhase() {
         fi
 
         local flagsArray=(
-            -j$buildCores -l$NIX_BUILD_CORES
+            -j$buildCores
             $ninjaFlags "${ninjaFlagsArray[@]}"
             $checkTarget
         )

diff --git a/pkgs/games/uhexen2/default.nix b/pkgs/games/uhexen2/default.nix
@@ -30,7 +30,7 @@ stdenv.mkDerivation rec {
     runHook preBuild
     for makefile in "''${makeFiles[@]}"; do
           local flagsArray=(
-            -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+            -j$NIX_BUILD_CORES
             SHELL=$SHELL
             $makeFlags "''${makeFlagsArray[@]}"
             $buildFlags "''${buildFlagsArray[@]}"

diff --git a/pkgs/games/xonotic/default.nix b/pkgs/games/xonotic/default.nix
@@ -83,14 +83,14 @@ let
     '';
 
     buildPhase = (lib.optionalString withDedicated ''
-      make -j $NIX_BUILD_CORES -l $NIX_BUILD_CORES sv-${target}
+      make -j $NIX_BUILD_CORES sv-${target}
     '' + lib.optionalString withGLX ''
-      make -j $NIX_BUILD_CORES -l $NIX_BUILD_CORES cl-${target}
+      make -j $NIX_BUILD_CORES cl-${target}
     '' + lib.optionalString withSDL ''
-      make -j $NIX_BUILD_CORES -l $NIX_BUILD_CORES sdl-${target}
+      make -j $NIX_BUILD_CORES sdl-${target}
     '') + ''
       pushd ../d0_blind_id
-      make -j $NIX_BUILD_CORES -l $NIX_BUILD_CORES
+      make -j $NIX_BUILD_CORES
       popd
     '';
 

diff --git a/pkgs/os-specific/linux/trace-cmd/default.nix b/pkgs/os-specific/linux/trace-cmd/default.nix
@@ -35,7 +35,7 @@ stdenv.mkDerivation rec {
   # because the Makefile would not print warnings about too old
   # libraries (see "warning:" in the Makefile)
   postBuild = ''
-    make libs doc -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+    make libs doc -j$NIX_BUILD_CORES
   '';
 
   installTargets = [

diff --git a/pkgs/stdenv/generic/setup.sh b/pkgs/stdenv/generic/setup.sh
@@ -1100,7 +1100,7 @@ buildPhase() {
         # Old bash empty array hack
         # shellcheck disable=SC2086
         local flagsArray=(
-            ${enableParallelBuilding:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}
+            ${enableParallelBuilding:+-j${NIX_BUILD_CORES}}
             SHELL=$SHELL
             $makeFlags "${makeFlagsArray[@]}"
             $buildFlags "${buildFlagsArray[@]}"
@@ -1139,7 +1139,7 @@ checkPhase() {
         # Old bash empty array hack
         # shellcheck disable=SC2086
         local flagsArray=(
-            ${enableParallelChecking:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}
+            ${enableParallelChecking:+-j${NIX_BUILD_CORES}}
             SHELL=$SHELL
             $makeFlags "${makeFlagsArray[@]}"
             ${checkFlags:-VERBOSE=y} "${checkFlagsArray[@]}"
@@ -1273,7 +1273,7 @@ installCheckPhase() {
         # Old bash empty array hack
         # shellcheck disable=SC2086
         local flagsArray=(
-            ${enableParallelChecking:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}
+            ${enableParallelChecking:+-j${NIX_BUILD_CORES}}
             SHELL=$SHELL
             $makeFlags "${makeFlagsArray[@]}"
             $installCheckFlags "${installCheckFlagsArray[@]}"

diff --git a/pkgs/tools/admin/tigervnc/default.nix b/pkgs/tools/admin/tigervnc/default.nix
@@ -92,7 +92,7 @@ stdenv.mkDerivation rec {
         --with-xkb-path=${xkeyboard_config}/share/X11/xkb \
         --with-xkb-bin-directory=${xorg.xkbcomp}/bin \
         --with-xkb-output=$out/share/X11/xkb/compiled
-    make TIGERVNC_SRC=$src TIGERVNC_BUILDDIR=`pwd`/../.. -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+    make TIGERVNC_SRC=$src TIGERVNC_BUILDDIR=`pwd`/../.. -j$NIX_BUILD_CORES
     popd
   '' + lib.optionalString stdenv.isDarwin ''
     make dmg

diff --git a/pkgs/tools/security/tracee/default.nix b/pkgs/tools/security/tracee/default.nix
@@ -63,7 +63,7 @@ buildGoModule rec {
 
   buildPhase = ''
     runHook preBuild
-    make $makeFlags ''${enableParallelBuilding:+-j$NIX_BUILD_CORES -l$NIX_BUILD_CORES}
+    make $makeFlags ''${enableParallelBuilding:+-j$NIX_BUILD_CORES
     runHook postBuild
   '';
 

diff --git a/pkgs/tools/typesetting/tex/texlive/bin.nix b/pkgs/tools/typesetting/tex/texlive/bin.nix
@@ -202,7 +202,7 @@ core-big = stdenv.mkDerivation { #TODO: upmendex
         if [[ "$path" =~ "libs/pplib" ]]; then
           # TODO: revert for texlive 2022
           # ../../../texk/web2c/luatexdir/luamd5/md5lib.c:197:10: fatal error: 'utilsha.h' file not found
-          make ''${enableParallelBuilding:+-j''${NIX_BUILD_CORES} -l''${NIX_BUILD_CORES}}
+          make ''${enableParallelBuilding:+-j''${NIX_BUILD_CORES}}
         fi
       )
     done