test: add eval, smoke, VM and formatting checks

.github/workflows/build.yaml

@@ -20,16 +20,32 @@ jobs:
         with:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}
 
+      # ubuntu-latest advertises KVM (via /dev/kvm) so the NixOS VM tests can
+      # boot; the GitHub macOS runners cannot run Linux VMs, so the macOS job
+      # only runs the non-VM checks below.
+      - name: Advertise KVM and nixos-test build features
+        if: runner.os == 'Linux'
+        run: echo "system-features = kvm nixos-test" | sudo tee -a /etc/nix/nix.conf
+
       - name: Enable binary cache with Cachix
         uses: cachix/cachix-action@v17
         with:
           name: nix-relic
           # If you chose API tokens for write access OR if you have a private cache
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
 
-      - name: Run Nix checks
+      - name: Run Nix checks (Ubuntu - including VM tests)
+        if: runner.os == 'Linux'
         run: nix flake check
 
+      - name: Run Nix checks (macOS - excluding VM tests)
+        if: runner.os == 'macOS'
+        run: |
+          nix build --no-link '.#checks.aarch64-darwin.eval-tests' \
+            '.#checks.aarch64-darwin.smoke-nrdot-collector' \
+            '.#checks.aarch64-darwin.smoke-nri-flex' \
+            '.#checks.aarch64-darwin.format-check'
+
   build:
     needs: [checks]
     runs-on: ${{ matrix.os }}

checks/default.nix

@@ -0,0 +1,62 @@
+# Aggregates all flake checks.
+{
+  lib,
+  pkgs,
+  inputs,
+}:
+let
+  # A pkgs set that also contains this flake's packages. We merge the plain
+  # packages into `pkgs` instead of using `pkgs.extend` with the overlay, since
+  # our package set no longer references stdenv at the top level (that would
+  # cause infinite recursion when an overlay is applied).
+  #
+  # newrelic-fluent-bit-output is Linux-only, so it is filtered out elsewhere.
+  ourPackages = import ../pkgs { pkgs = pkgs; };
+  overlayPkgs =
+    pkgs
+    // (
+      if pkgs.stdenv.hostPlatform.isLinux then
+        ourPackages
+      else
+        builtins.removeAttrs ourPackages [ "newrelic-fluent-bit-output" ]
+    );
+
+  # Package set for the NixOS VM guests. The test driver runs on the host
+  # (nixpkgs supports aarch64-darwin hosts driving aarch64-linux guests via
+  # HVF), but the guest itself is always Linux, so the flake's packages must
+  # be the Linux builds (`pkgs.pkgsLinux`).
+  guestPkgs = pkgs.pkgsLinux // import ../pkgs { pkgs = pkgs.pkgsLinux; };
+in
+(import ./eval-tests.nix {
+  inherit lib;
+  pkgs = overlayPkgs;
+  plainPkgs = pkgs;
+})
+// (import ./smoke-tests.nix {
+  inherit lib;
+  pkgs = overlayPkgs;
+})
+// (import ./vm-tests.nix {
+  inherit lib;
+  pkgs = overlayPkgs;
+  inherit guestPkgs;
+})
+// {
+  # Ensures every .nix file in the repo is formatted with the flake's formatter
+  format-check =
+    pkgs.runCommand "format-check"
+      {
+        files = builtins.filter (f: lib.hasSuffix ".nix" f) (lib.filesystem.listFilesRecursive ../.);
+        formatter = "${pkgs.nixfmt}/bin/nixfmt";
+      }
+      ''
+        for f in $files; do
+              $formatter - <"$f" | cmp -s - "$f" || {
+                echo "NOT FORMATTED: $f"
+                echo "Run 'nix fmt' and commit the changes."
+                exit 1
+              }
+            done
+            touch $out
+      '';
+}

checks/eval-tests.nix

@@ -0,0 +1,264 @@
+# Unit tests that evaluate the modules and assert on the resulting config.
+# These run quickly and don't build any packages.
+{
+  lib,
+  pkgs,
+  plainPkgs,
+}:
+let
+  inherit (lib) types mkOption runTests;
+
+  # Minimal stub options so the NixOS modules can be evaluated standalone
+  moduleStubs = {
+    options = {
+      environment.etc = mkOption {
+        type = types.attrsOf types.anything;
+        default = { };
+      };
+      systemd.services = mkOption {
+        type = types.attrsOf types.anything;
+        default = { };
+      };
+      launchd.daemons = mkOption {
+        type = types.attrsOf types.anything;
+        default = { };
+      };
+      assertions = mkOption {
+        type = types.listOf (
+          types.submodule {
+            options = {
+              assertion = mkOption {
+                type = types.bool;
+              };
+              message = mkOption {
+                type = types.str;
+              };
+            };
+          }
+        );
+        default = [ ];
+      };
+    };
+  };
+
+  mkEval =
+    {
+      module,
+      pkg ? plainPkgs,
+      config,
+    }:
+    lib.evalModules {
+      modules = [
+        moduleStubs
+        module
+        config
+      ];
+      specialArgs = {
+        pkgs = pkg;
+      };
+    };
+
+  infraModule = import ../modules/nixos/newrelic-infra.nix;
+  nrdotModule = import ../modules/nixos/nrdot-collector.nix;
+
+  infraWith =
+    {
+      pkg ? plainPkgs,
+      config ? { },
+    }:
+    (mkEval {
+      module = infraModule;
+      inherit pkg;
+      config = {
+        services.newrelic-infra = {
+          enable = true;
+        }
+        // config;
+      };
+    }).config.services.newrelic-infra;
+
+  infraConfigWith =
+    {
+      pkg ? plainPkgs,
+      config ? { },
+    }:
+    (mkEval {
+      module = infraModule;
+      inherit pkg;
+      config = {
+        services.newrelic-infra = {
+          enable = true;
+        }
+        // config;
+      };
+    }).config;
+
+  # Only meaningful on Linux: the fluent-bit output plugin is Linux-only
+  linuxOnlyTests = lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
+    testNixosInfraLoggingNrLibWithOverlay = {
+      expr =
+        (infraWith {
+          pkg = pkgs;
+          config = {
+            logging = [
+              {
+                name = "test";
+                file = "/var/log/test.log";
+              }
+            ];
+          };
+        }).settings.fluent_bit_nr_lib_path;
+      expected = "${pkgs.newrelic-fluent-bit-output}/lib/out_newrelic.so";
+    };
+
+    # The service runs with system utilities (kmod is Linux-only) on its PATH
+    testNixosInfraPathIncludesKmod = {
+      expr = lib.elem pkgs.kmod (infraConfigWith { }).systemd.services.newrelic-infra.path;
+      expected = true;
+    };
+  };
+
+  tests = {
+    # The service is defined and uses the agent's service binary
+    testNixosInfraService = {
+      expr = (infraConfigWith { }).systemd.services.newrelic-infra.serviceConfig.ExecStart;
+      expected = "${pkgs.infrastructure-agent}/bin/newrelic-infra-service -config ${
+        (pkgs.formats.yaml { }).generate "config.yaml" (infraWith { }).settings
+      }";
+    };
+
+    # Regression test for https://github.com/DavSanchez/Nix-Relic/issues/88:
+    # the agent hardcodes FHS paths like /sbin/lsmod
+    testNixosInfraLsmodBindPaths = {
+      expr = (infraConfigWith { }).systemd.services.newrelic-infra.serviceConfig.BindReadOnlyPaths;
+      expected = [
+        "/run/current-system/sw/bin:/sbin"
+        "/run/current-system/sw/bin/systemctl:/bin/systemctl"
+      ];
+    };
+
+    # The package option defaults to nixpkgs' infrastructure-agent
+    testNixosInfraPackageDefault = {
+      expr = (infraWith { }).package;
+      expected = pkgs.infrastructure-agent;
+    };
+
+    # Log forwarding wires the fluent-bit executable into the agent settings
+    testNixosInfraLoggingFluentBitPath = {
+      expr =
+        (infraWith {
+          config = {
+            logging = [
+              {
+                name = "test";
+                file = "/var/log/test.log";
+              }
+            ];
+          };
+        }).settings.fluent_bit_exe_path;
+      expected = "${pkgs.fluent-bit}/bin/fluent-bit";
+    };
+
+    # Without the overlay there is no output plugin package
+    testNixosInfraLoggingNrLibWithoutOverlay = {
+      expr =
+        (infraWith {
+          config = {
+            logging = [
+              {
+                name = "test";
+                file = "/var/log/test.log";
+              }
+            ];
+          };
+        }).fluentBitPackage;
+      expected = null;
+    };
+
+    # The logging config file is dropped into logging.d
+    testNixosInfraLoggingFileGenerated = {
+      expr =
+        (infraConfigWith {
+          config = {
+            logging = [
+              {
+                name = "test";
+                file = "/var/log/test.log";
+              }
+            ];
+          };
+        }).environment.etc ? "newrelic-infra/logging.d/logging.yml";
+      expected = true;
+    };
+
+    # No logging config file is generated when logging is disabled
+    testNixosInfraNoLoggingFileWithoutLogs = {
+      expr = (infraConfigWith { }).environment.etc ? "newrelic-infra/logging.d/logging.yml";
+      expected = false;
+    };
+
+    # Integrations are written to integrations.d
+    testNixosInfraIntegrationsFileGenerated = {
+      expr =
+        (infraConfigWith {
+          config = {
+            integrations = [
+              {
+                name = "flex";
+                package = pkgs.nri-flex;
+              }
+            ];
+          };
+        }).environment.etc ? "newrelic-infra/integrations.d/flex.yml";
+      expected = true;
+    };
+
+    # The nrdot-collector package is only available through the overlay
+    testNixosNrdotCollectorPackageFromOverlay = {
+      expr =
+        (mkEval {
+          module = nrdotModule;
+          pkg = pkgs;
+          config = {
+            services.nrdot-collector = {
+              enable = true;
+              settings = { };
+            };
+          };
+        }).config.services.nrdot-collector.package;
+      expected = pkgs.nrdot-collector;
+    };
+
+    # The nrdot-collector service is defined with the expected binary
+    testNixosNrdotCollectorService = {
+      expr =
+        (mkEval {
+          module = nrdotModule;
+          pkg = pkgs;
+          config = {
+            services.nrdot-collector = {
+              enable = true;
+              settings = { };
+            };
+          };
+        }).config.systemd.services.nrdot-collector.serviceConfig.ExecStart;
+      expected = "${pkgs.nrdot-collector}/bin/nrdot-collector --config=file:${
+        (pkgs.formats.yaml { }).generate "config.yaml" { }
+      }";
+    };
+  };
+
+  failures = runTests (tests // linuxOnlyTests);
+in
+{
+  eval-tests = pkgs.runCommand "eval-tests" { } (
+    ''
+      ${lib.concatMapStringsSep "\n" (failure: ''
+        echo "FAIL: ${failure.name}"
+        echo "  expected: ${lib.generators.toPretty { } failure.expected}"
+        echo "  result:   ${lib.generators.toPretty { } failure.result}"
+      '') failures}
+    ''
+    + (if failures == [ ] then "touch $out" else "exit 1")
+  );
+}