Add reliable local test runner and repair workspace and LIMO fixtures

README.md

@@ -15,7 +15,32 @@ git submodule update --init --recursive --remote
 git pull --recurse-submodules
 ```
 
-## Running test locally
+## Running tests locally with installed plugins
+
+From this repository root, use the local runner without opening the MATLAB project:
+
+```matlab
+results = run_eeglab_tests;
+assertSuccess(results);
+```
+
+The runner discovers the root sample data wrapper, all `unittesting_*` test folders except helpers, and `regression_tests`. It does not execute project startup installers, nested runners, or the legacy email reporting harness. Install the required plugins and local test datasets before running it. EEGLAB startup may query the plugin server, but the runner does not upgrade plugins.
+
+Use a fresh MATLAB process and a disposable checkout or copy. Legacy tests write derived data in the checkout. The runner restores the working directory, MATLAB path, default figure visibility, and EEGLAB preference file. The menu and LIMO wrappers additionally restore their base workspace and random state. Do not run multiple suites concurrently against the same data or preference file.
+
+Each run saves `manifest.csv`, source path information in `suite.mat`, MATLAB results, a CSV summary, JUnit XML, and individual completed results. `current.txt` identifies the active test if execution stalls. Results are retained under a new `test-results` directory by default. An empty selection is an error, not a passing run.
+
+```matlab
+run_eeglab_tests('DiscoverOnly', true);
+results = run_eeglab_tests('IncludeLimo', false);
+results = run_eeglab_tests('Name', '*pop_selectevent*');
+```
+
+LIMO is included by default. Its two integration cases require the local `ds002718` dataset and can take substantially longer than the other tests. They create new temporary output directories and print their locations. Existing results are never deleted. To choose explicit output directories, call `limo_preproc_stats_hw(datasetPath, newOutputPath)` or `limo_test_integration(datasetPath, newOutputPath)` with the helper folders on the MATLAB path. The integration case saves `first_level_results.mat`. Use `limo_test_second_level(preparationFile)` to rerun the second level cases into another new directory. That directory retains `integration_results.mat`, including the nine internal group outcomes and full exception objects. A prerequisite failure is identified separately from an analysis failure.
+
+A passing legacy wrapper does not necessarily contain a numerical assertion. The manifest records executed cases, not a guarantee of complete scientific coverage.
+
+## Running registered project tests
 
 Open the EEGLAB_test.prj project, then copy and paste the following code.
 

eeglab_test_suite.m

@@ -0,0 +1,26 @@
+function suite = eeglab_test_suite(root, includeLimo)
+% Discover local tests explicitly, without opening the MATLAB project.
+% Project startup installs plugins. The local runner must use installed code.
+if nargin < 1
+    root = fileparts(mfilename('fullpath'));
+end
+if nargin < 2
+    includeLimo = true;
+end
+suite = matlab.unittest.TestSuite.fromFile(fullfile(root, 'eeglab_tests_wrapperTest.m'));
+folders = dir(fullfile(root, 'unittesting_*'));
+for index = 1:numel(folders)
+    name = folders(index).name;
+    if ~folders(index).isdir || strcmp(name, 'unittesting_common') || ...
+            (~includeLimo && strcmp(name, 'unittesting_limo'))
+        continue
+    end
+    suite = [suite matlab.unittest.TestSuite.fromFolder(fullfile(root, name), ...
+        'IncludingSubfolders', true)]; %#ok<AGROW>
+end
+suite = [suite matlab.unittest.TestSuite.fromFolder(fullfile(root, 'regression_tests'), ...
+    'IncludingSubfolders', true)];
+keys = string({suite.Name}) + "|" + string({suite.BaseFolder});
+[~, index] = unique(keys, 'stable');
+suite = suite(index);
+end

regression_tests/test_limo_test_helpers.m

@@ -0,0 +1,40 @@
+function tests = test_limo_test_helpers
+tests = functiontests(localfunctions);
+end
+
+function setupOnce(testCase)
+root = fileparts(fileparts(mfilename('fullpath')));
+testCase.applyFixture(matlab.unittest.fixtures.PathFixture(fullfile(root, 'unittesting_limo')));
+end
+
+function testRejectsExistingOutput(testCase)
+folder = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture).Folder;
+handle = fopen(fullfile(folder, 'participants.tsv'), 'w');
+fprintf(handle, 'participant_id\n');
+fclose(handle);
+verifyError(testCase, @() limo_test_output(folder, folder), '');
+verifyTrue(testCase, isfile(fullfile(folder, 'participants.tsv')));
+end
+
+function testRejectsMissingDataset(testCase)
+folder = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture).Folder;
+verifyError(testCase, @() limo_test_output(folder, fullfile(folder, 'output')), '');
+verifyFalse(testCase, isfolder(fullfile(folder, 'output')));
+end
+
+function testListsFollowReturnedPathsAndGroupOrder(testCase)
+folder = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture).Folder;
+STUDY.datasetinfo = struct('group', {'B', 'A', 'B'});
+files.mat = cell(3,1); files.Beta = cell(3,1); files.con = cell(3,1);
+for subject = 1:3
+    file = fullfile(folder, sprintf('arbitrary_subject_%d.mat', subject));
+    save(file, 'subject');
+    files.mat{subject} = file;
+    files.Beta{subject} = file;
+    files.con{subject} = {file};
+end
+lists = limo_test_lists(files, STUDY, fullfile(folder, 'lists'));
+verifyEqual(testCase, readlines(lists.beta, 'EmptyLineRule', 'skip'), string(files.Beta));
+verifyEqual(testCase, readlines(lists.group_con{1}, 'EmptyLineRule', 'skip'), string(files.Beta([1 3])));
+verifyEqual(testCase, readlines(lists.group_beta{2}, 'EmptyLineRule', 'skip'), string(files.Beta(2)));
+end

regression_tests/test_local_discovery.m

@@ -0,0 +1,21 @@
+function tests = test_local_discovery
+tests = functiontests(localfunctions);
+end
+
+function testNoInstallersOrNestedRunners(testCase)
+root = fileparts(fileparts(mfilename('fullpath')));
+testCase.applyFixture(matlab.unittest.fixtures.PathFixture(root));
+suite = eeglab_test_suite(root, false);
+names = string({suite.Name});
+verifyEqual(testCase, numel(unique(names)), numel(names));
+verifyFalse(testCase, any(contains(names, {'test_add_plugins', 'example_local_test', 'runtest/'})));
+verifyTrue(testCase, any(names == "adminfunc_eeglab_execmenu_wrapperTest/test_i_pass_general"));
+verifyFalse(testCase, any(startsWith(names, "test_eeglab_execmenu/")));
+verifyTrue(testCase, any(startsWith(names, "test_workspace_fixture/")));
+end
+
+function testLimoWrappersAreDiscoverable(testCase)
+root = fileparts(fileparts(mfilename('fullpath')));
+suite = matlab.unittest.TestSuite.fromFile(fullfile(root, 'unittesting_limo', 'limo_wrapperTest.m'));
+verifyEqual(testCase, sort(string({suite.Name})), sort(["limo_wrapperTest/test_preprocessing", "limo_wrapperTest/test_integration"]));
+end

regression_tests/test_workspace_fixture.m

@@ -0,0 +1,54 @@
+function tests = test_workspace_fixture
+tests = functiontests(localfunctions);
+end
+
+function setupOnce(testCase)
+root = fileparts(fileparts(mfilename('fullpath')));
+testCase.applyFixture(matlab.unittest.fixtures.PathFixture(fullfile(root, 'unittesting_common')));
+end
+
+function testRestoresStateOnSuccess(testCase)
+checkRestore(testCase, false);
+end
+
+function testRestoresStateOnError(testCase)
+checkRestore(testCase, true);
+end
+
+function testPreservesExistingEeglabWindow(testCase)
+original = figure('Visible', 'off', 'Tag', 'EEGLAB', 'UserData', 42);
+cleanup = onCleanup(@() delete(original(isgraphics(original)))); %#ok<NASGU>
+eeglab_test_workspace('delete(findall(groot, ''Type'', ''figure'', ''Tag'', ''EEGLAB'')); figure(''Tag'', ''EEGLAB'');');
+verifyTrue(testCase, isgraphics(original));
+verifyEqual(testCase, get(original, 'Tag'), 'EEGLAB');
+verifyEqual(testCase, get(original, 'UserData'), 42);
+end
+
+function checkRestore(testCase, shouldError)
+name = 'EEGLAB_FIXTURE_SENTINEL';
+previous = evalin('base', 'whos(''EEGLAB_FIXTURE_SENTINEL'')');
+if ~isempty(previous), oldValue = evalin('base', name); else, oldValue = []; end
+cleanup = onCleanup(@() restoreSentinel(previous, oldValue)); %#ok<NASGU>
+assignin('base', name, 42);
+oldFolder = pwd;
+oldRandom = rng;
+oldFigures = findall(groot, 'Type', 'figure');
+commands = 'EEGLAB_FIXTURE_SENTINEL = 99; rng(71); figure; fid=fopen(''fixture.txt'',''w''); fclose(fid);';
+if shouldError
+    verifyError(testCase, @() eeglab_test_workspace([commands ' error(''EEGLAB:testFailure'',''expected'');']), 'EEGLAB:testFailure');
+else
+    eeglab_test_workspace(commands);
+end
+verifyEqual(testCase, evalin('base', name), 42);
+verifyEqual(testCase, pwd, oldFolder);
+verifyEqual(testCase, rng, oldRandom);
+verifyEqual(testCase, findall(groot, 'Type', 'figure'), oldFigures);
+end
+
+function restoreSentinel(previous, value)
+evalin('base', 'clear EEGLAB_FIXTURE_SENTINEL');
+if ~isempty(previous)
+    if previous.global, evalin('base', 'global EEGLAB_FIXTURE_SENTINEL'); end
+    assignin('base', 'EEGLAB_FIXTURE_SENTINEL', value);
+end
+end

run_eeglab_tests.m

@@ -0,0 +1,69 @@
+function results = run_eeglab_tests(varargin)
+% Run installed EEGLAB tests locally, with a manifest and persistent results.
+% run_eeglab_tests('OutputDirectory', folder, 'IncludeLimo', false)
+% run_eeglab_tests('DiscoverOnly', true)
+% Legacy tests write data in this checkout. Use a disposable copy if needed.
+parser = inputParser;
+parser.addParameter('OutputDirectory', '', @(x) ischar(x) || isstring(x));
+parser.addParameter('IncludeLimo', true, @(x) islogical(x) && isscalar(x));
+parser.addParameter('DiscoverOnly', false, @(x) islogical(x) && isscalar(x));
+parser.addParameter('Name', '*', @(x) ischar(x) || isstring(x));
+parser.parse(varargin{:});
+options = parser.Results;
+root = fileparts(mfilename('fullpath'));
+output = char(options.OutputDirectory);
+if isempty(output)
+    output = fullfile(root, 'test-results', char(datetime('now', 'Format', 'yyyyMMdd_HHmmss_SSS')));
+end
+if ~isfolder(output)
+    mkdir(output);
+end
+% Convert relative paths before legacy tests change the working directory.
+[ok, attributes] = fileattrib(output);
+assert(ok, 'Cannot resolve output directory.');
+output = attributes.Name;
+assert(~isfile(fullfile(output, 'manifest.csv')), 'Choose a new output directory to preserve previous results.');
+oldPath = path;
+oldFolder = pwd;
+oldVisibility = get(groot, 'DefaultFigureVisible');
+cleanup = onCleanup(@() restoreRunner(oldPath, oldFolder, oldVisibility)); %#ok<NASGU>
+cd(root);
+addpath(root, fullfile(root, 'eeglab'), fullfile(root, 'regression_tests'));
+addpath(genpath(fullfile(root, 'unittesting_common')));
+set(groot, 'DefaultFigureVisible', 'off');
+eeglab;
+preferences = eeglab_test_preferences; %#ok<NASGU>
+suite = eeglab_test_suite(root, options.IncludeLimo);
+pattern = ['^' regexptranslate('wildcard', char(options.Name)) '$'];
+suite = suite.selectIf(matlab.unittest.selectors.HasName(matlab.unittest.constraints.Matches(pattern)));
+assert(~isempty(suite), 'EEGLAB:NoTests', 'No tests matched the requested selection.');
+manifest = table(string({suite.Name})', string({suite.BaseFolder})', ...
+    'VariableNames', {'Name', 'BaseFolder'});
+writetable(manifest, fullfile(output, 'manifest.csv'));
+environment = struct('matlab', version, 'architecture', computer, ...
+    'eeglab', which('eeglab'), 'version', eeg_getversion, 'path', path);
+save(fullfile(output, 'suite.mat'), 'suite', 'environment');
+fprintf('Discovered %d unique tests. Results: %s\n', numel(suite), output);
+results = matlab.unittest.TestResult.empty;
+if options.DiscoverOnly
+    return
+end
+runner = matlab.unittest.TestRunner.withTextOutput;
+runner.addPlugin(matlab.unittest.plugins.XMLPlugin.producingJUnitFormat(fullfile(output, 'results.xml')));
+runner.addPlugin(matlab.unittest.plugins.DiagnosticsRecordingPlugin);
+runner.addPlugin(EeglabTestProgressPlugin(output));
+started = datetime('now');
+results = runner.run(suite);
+finished = datetime('now');
+save(fullfile(output, 'results.mat'), 'results', 'started', 'finished', 'environment');
+summary = table(results);
+writetable(summary(:, {'Name', 'Passed', 'Failed', 'Incomplete', 'Duration'}), fullfile(output, 'results.csv'));
+fprintf('Completed %d tests: %d passed, %d failed, %d incomplete.\n', ...
+    numel(results), nnz([results.Passed]), nnz([results.Failed]), nnz([results.Incomplete]));
+end
+
+function restoreRunner(oldPath, oldFolder, oldVisibility)
+cd(oldFolder);
+path(oldPath);
+set(groot, 'DefaultFigureVisible', oldVisibility);
+end

unittesting_adminfunc/eeglab_execmenu/adminfunc_eeglab_execmenu_wrapperTest.m

@@ -1,5 +1,9 @@
 function tests = adminfunc_eeglab_execmenu_wrapperTest
 tests = functiontests(localfunctions);
 
+function setupOnce(testCase)
+root = fileparts(fileparts(fileparts(mfilename('fullpath'))));
+testCase.applyFixture(matlab.unittest.fixtures.PathFixture(fullfile(root, 'unittesting_common')));
+
 function test_i_pass_general(~)
-evalin('base', 'test_eeglab_execmenu');
+test_eeglab_execmenu;

unittesting_adminfunc/eeglab_execmenu/eeglab_execmenu_commands.m

@@ -0,0 +1,23 @@
+%%
+eeglab
+eeglabp = fileparts(which('eeglab'));
+tmpdata = rand(10,1000);
+eeglab_execmenu('From ASCII/float file or MATLAB array', 'pop_importdata', { 'dataformat','array','nbchan',0,'data','tmpdata','srate',100 });
+assert(EEG.nbchan == 10 && EEG.pnts == 1000 && EEG.srate == 100);
+assert(max(abs(double(EEG.data(:))-tmpdata(:))) < 1e-6);
+eeglab_execmenu('Save current dataset as', 'pop_saveset', { 'test.set' });
+eeglab_execmenu('Dataset info', 'pop_editset', { 'subject', 'test2' });
+assert(strcmp(EEG.subject, 'test2'));
+eeglab_execmenu('Resave current dataset(s)', 'pop_saveset', { 'savemode' 'resave' });
+eeglab_execmenu('Load existing dataset', 'pop_loadset', { fullfile(eeglabp, 'sample_data', 'eeglab_data.set') });
+eeglab_execmenu('Event values', 'pop_editeventvals', { 'changefield',{1,'position',3} });
+eeglab_execmenu('About this dataset', 'pop_comments', { strvcat('EEGLAB Tutorial Dataset','test') });
+eeglab_execmenu('Channel locations', 'pop_chanedit', { 'lookup', fullfile(eeglabp, 'plugins', 'dipfit', 'standard_BEM', 'elec', 'standard_1005.elc') });
+eeglab_execmenu('Change sampling rate', 'pop_resample', { 64 });
+assert(EEG.srate == 64);
+
+SAVEDCOM = ALLCOM;
+for iCommand = length(SAVEDCOM):-1:1
+    evalin('base', SAVEDCOM{iCommand});
+end
+assert(EEG.srate == 64);

unittesting_adminfunc/eeglab_execmenu/test_eeglab_execmenu.m

@@ -1,18 +1,5 @@
-%%
-eeglab
-eeglabp = fileparts(which('eeglab'));
-tmpdata = rand(10,1000);
-eeglab_execmenu('From ASCII/float file or MATLAB array', 'pop_importdata', { 'dataformat','array','nbchan',0,'data','tmpdata','srate',100 });
-eeglab_execmenu('Save current dataset as', 'pop_saveset', { 'test.set' });
-eeglab_execmenu('Dataset info', 'pop_editset', { 'subject', 'test2' });
-eeglab_execmenu('Resave current dataset(s)', 'pop_saveset', { 'savemode' 'resave' });
-eeglab_execmenu('Load existing dataset', 'pop_loadset', { fullfile(eeglabp, 'sample_data', 'eeglab_data.set') });
-eeglab_execmenu('Event values', 'pop_editeventvals', { 'changefield',{1,'position',3} });
-eeglab_execmenu('About this dataset', 'pop_comments', { strvcat('EEGLAB Tutorial Dataset','test') });
-eeglab_execmenu('Channel locations', 'pop_chanedit', { 'lookup', fullfile(eeglabp, 'plugins', 'dipfit', 'standard_BEM', 'elec', 'standard_1005.elc') });
-eeglab_execmenu('Change sampling rate', 'pop_resample', { 64 });
-
-SAVEDCOM = ALLCOM;
-for iCommand = length(SAVEDCOM):-1:1
-    evalin('base', SAVEDCOM{iCommand});
+function test_eeglab_execmenu
+% Menu callbacks execute in the base workspace, including history replay.
+folder = fileparts(mfilename('fullpath'));
+eeglab_test_workspace(fileread(fullfile(folder, 'eeglab_execmenu_commands.m')));
 end

unittesting_common/EeglabTestProgressPlugin.m

@@ -0,0 +1,31 @@
+classdef EeglabTestProgressPlugin < matlab.unittest.plugins.TestRunnerPlugin
+    % Preserve completed results and identify a stalled test before completion.
+    properties (Access = private)
+        OutputDirectory
+        Count = 0
+    end
+    methods
+        function plugin = EeglabTestProgressPlugin(folder)
+            plugin.OutputDirectory = folder;
+            mkdir(fullfile(folder, 'individual'));
+        end
+    end
+    methods (Access = protected)
+        function runTest(plugin, data)
+            file = fullfile(plugin.OutputDirectory, 'current.txt');
+            handle = fopen(file, 'w');
+            assert(handle ~= -1, 'Cannot write test progress to %s.', file);
+            cleanup = onCleanup(@() fclose(handle));
+            fprintf(handle, '%s\n%s\n', char(data.Name), char(datetime('now')));
+            clear cleanup
+            runTest@matlab.unittest.plugins.TestRunnerPlugin(plugin, data);
+        end
+        function reportFinalizedResult(plugin, data)
+            result = data.TestResult;
+            plugin.Count = plugin.Count + 1;
+            save(fullfile(plugin.OutputDirectory, 'individual', ...
+                sprintf('%04d.mat', plugin.Count)), 'result');
+            reportFinalizedResult@matlab.unittest.plugins.TestRunnerPlugin(plugin, data);
+        end
+    end
+end

unittesting_common/eeglab_test_preferences.m

@@ -0,0 +1,21 @@
+function cleanup = eeglab_test_preferences
+% Restore the user's EEGLAB option file after tests that call pop_editoptions.
+eeglab_options;
+file = fullfile(homefolder, 'eeg_options.m');
+existed = isfile(file);
+backup = tempname;
+if existed
+    copyfile(file, backup);
+end
+cleanup = onCleanup(@() restorePreferences(file, backup, existed));
+end
+
+function restorePreferences(file, backup, existed)
+if existed
+    copyfile(backup, file);
+    delete(backup);
+elseif isfile(file)
+    delete(file);
+end
+clear eeg_options
+end