diff --git a/docs/WINDOWS-COLLECTOR-RELIABILITY.md b/docs/WINDOWS-COLLECTOR-RELIABILITY.md new file mode 100644 index 0000000..99c448f --- /dev/null +++ b/docs/WINDOWS-COLLECTOR-RELIABILITY.md @@ -0,0 +1,123 @@ +# Windows collector reliability validation — 2026-10-30 + +## Root causes + +The supplied Server 2025 assessment reported tasks failed and 16 certificate +errors, but its generic errors did not retain exception types. Read-only native +reproduction established two defects: + +* Tasks: one action executable made .NET Framework `Path.GetFileName` throw a + `MethodInvocationException` wrapping `ArgumentException`. The projection was + inside a single assignment/query catch, so that one value discarded all tasks. +* Certificates: the Windows certificate provider returned store `PSPath` values + such as `Microsoft.PowerShell.Security\Certificate::My`, without LocalMachine. + Using those paths produced `ItemNotFoundException` / `ObjectNotFound` for each + store. Store-list success alone incorrectly gave partial credit when every + store query failed. + +The original code already enumerated stores separately; recursion was not the +cause. Missing modules and JSON conversion were checked as failure paths, but +were not the reproduced causes on this host. + +## Changes + +Executable basenames now use string operations and the existing strict allowlist, +avoiding filesystem path validation. Malformed values become an omission marker; +task records remain visible. Each task is projected and JSON-checked separately. +Principal group accounts are used when UserId is absent. Arguments, XML, inline +scripts, event queries and credentials are never projected. + +Certificates use explicit `Cert:\LocalMachine\` paths. Each store query +and certificate projection is isolated; empty successful stores receive credit, +inaccessible stores produce errors, and listing names alone does not give credit +when all listed stores fail. Only subject, issuer, thumbprint, expiration and +store are collected. No private key or certificate export APIs are used. + +Read helpers promote nonterminating errors to terminating errors. Per-record JSON +checks reject unrepresentable records. Final JSON failures exit with code 3 and +Python records `serialization_failed`, never a successful empty collection. +Existing subprocess timeout, UTF-8, output limits and schema validation remain. + +Optional `-Diagnostic` / Python `--diagnostic` adds exception class and category to +existing sanitized error messages. Fixed command stages appear in error scope. +No raw messages, task names, arguments, store names, targets, stack traces or +error identifiers are included in diagnostics. Inventory remains sensitive and +should be stored privately. Task/certificate UI detail pages display these error +scopes, codes and messages; Collection checks and reports retain their contract. + +The production Python backend deliberately continues to use trusted Windows +PowerShell 5.1. Direct PowerShell 7 invocation also includes the inbox Windows +module directory, without opening module lookup to user directories. Commands +autoload from those inbox module paths; missing commands are explicit failures. +No scoring policy, Linux collector or other inventory module was changed. + +## Results + +On the local Windows Server 2025 host, native 5.1 collection returned: + +| Collector | Status | Records | Errors | +| --- | --- | ---: | ---: | +| Tasks | complete | 192 (46 disabled) | 0 | +| Certificates | complete | 49 across 7 populated stores | 0 | + +Full Python subprocess collection passed schema validation, returned the same +record counts, and completed all nine checks. Evidence coverage rose from the +supplied assessment's 81% to 100%. Confidence remained moderate under the existing +classification policy; complete collection does not establish retirement safety. + +`python -m unittest discover -s tests -v`: **64 tests, 62 passed, 2 skipped**. +Skips: the Linux-only subprocess test on Windows and PowerShell 7 (not installed). +UI navigation/history/export tests, Linux parsing tests, scoring regressions, +Windows subprocess limits and schema tests passed. Native PowerShell AST and +synthetic tests passed under 5.1. Added cases cover normal/disabled/empty tasks, +malformed actions, per-task failures, nonterminating query denial, multiple and +empty stores, partial/all-store denial, redacted diagnostics and serialization +failures. A dedicated regression checks lower coverage/confidence without risk +inflation for failed tasks and partial certificates. + +## Exact validation commands + +Run from the repository root in an authorized elevated Windows PowerShell console. +These commands only read inventory; they do not change Server configuration or +execution policy. Choose a new output path for each saved inventory. + +```powershell +$PSVersionTable.PSVersion +powershell.exe -NoLogo -NoProfile -NonInteractive -File .\tests\powershell\Test-Collectors.ps1 +python -m unittest discover -s tests -v + +foreach ($section in @('tasks', 'certificates')) { + $raw = & powershell.exe -NoLogo -NoProfile -NonInteractive -File .\powershell\Get-ServerInventory.ps1 -Section $section -Diagnostic + if ($LASTEXITCODE -ne 0) { throw 'Collector process failed' } + $r = $raw | ConvertFrom-Json + [pscustomobject]@{ + Section = $section + Status = $r.status + Records = @($r.data).Count + Errors = $r.errors + } | ConvertTo-Json -Depth 5 + if ($r.status -ne 'complete') { throw 'Review incomplete collection' } + if (@($r.data).Count -eq 0) { throw 'No records: verify actual host contents independently' } +} + +python -m samsara.collectors.windows --diagnostic --output .\windows-validation-new.json +python .\tools\validate_windows_host.py --output-dir .\windows-validation-new + +# When PowerShell 7 is already installed: +pwsh.exe -NoLogo -NoProfile -NonInteractive -File .\tests\powershell\Test-Collectors.ps1 +pwsh.exe -NoLogo -NoProfile -NonInteractive -File .\powershell\Get-ServerInventory.ps1 -Section tasks -Diagnostic +pwsh.exe -NoLogo -NoProfile -NonInteractive -File .\powershell\Get-ServerInventory.ps1 -Section certificates -Diagnostic +``` + +The nonempty assertion above is a validation expectation for this host, not a +collector rule: legitimately empty successful queries remain complete. + +## Remaining limitations + +PowerShell 7 runtime execution and other Server releases were not available for +live validation. Tests run under each runtime when installed; 5.1 syntax and +Server 2016/2019/2022/2025 identity fixtures pass. Access restrictions may hide +objects even when an API reports success. A terminating failure of the initial +task enumeration cannot recover task registrations the API never returned, and +is reported failed. Inventory is a current observation, not historical activity. +No Windows Server configuration was modified. diff --git a/powershell/Get-ServerCertificates.ps1 b/powershell/Get-ServerCertificates.ps1 index 1b1883f..d2ac87e 100644 --- a/powershell/Get-ServerCertificates.ps1 +++ b/powershell/Get-ServerCertificates.ps1 @@ -4,12 +4,23 @@ function Get-SamsaraCertificates { Invoke-SamsaraRead $Result 'Local machine certificate store list' { $collection.stores = @(Get-ChildItem -LiteralPath 'Cert:\LocalMachine' -ErrorAction Stop) } + # Listing store names is not evidence that any listed store was readable. + if ($collection.stores.Count -gt 0) { $Result.successful_subchecks-- } foreach ($store in $collection.stores) { - Invoke-SamsaraRead $Result 'Local machine public certificate metadata' { - Get-ChildItem -LiteralPath $store.PSPath -ErrorAction Stop | ForEach-Object { - $Result.data += [pscustomobject]@{ subject = [string]$_.Subject; issuer = [string]$_.Issuer - expires = $_.NotAfter.ToUniversalTime().ToString('o'); thumbprint = [string]$_.Thumbprint - store = [string]$store.PSChildName } + # Certificate provider PSPath may omit LocalMachine (including on 5.1). + # Construct an explicit provider path from the enumerated store name. + $storeName = [string]$store.PSChildName + Invoke-SamsaraRead $Result 'Certificate store query' { + if ($storeName -notmatch '^[A-Za-z0-9_. -]+$') { throw 'Invalid store name' } + $certificates = @(Get-ChildItem -LiteralPath ('Cert:\LocalMachine\' + $storeName) -ErrorAction Stop) + foreach ($certificate in $certificates) { + Invoke-SamsaraRead $Result 'Certificate metadata projection' { + $row = [pscustomobject]@{ subject = [string]$certificate.Subject; issuer = [string]$certificate.Issuer + expires = $certificate.NotAfter.ToUniversalTime().ToString('o'); thumbprint = [string]$certificate.Thumbprint + store = $storeName } + $null = ConvertTo-Json -InputObject $row -Depth 4 -ErrorAction Stop + $Result.data += $row + } } } } diff --git a/powershell/Get-ServerInventory.ps1 b/powershell/Get-ServerInventory.ps1 index e0fe25f..e7aefed 100644 --- a/powershell/Get-ServerInventory.ps1 +++ b/powershell/Get-ServerInventory.ps1 @@ -1,12 +1,17 @@ #requires -Version 5.1 [CmdletBinding()] -param([ValidateSet('all','system','roles','services','network','tasks','software','storage','shares','certificates')][string]$Section = 'all') +param([ValidateSet('all','system','roles','services','network','tasks','software','storage','shares','certificates')][string]$Section = 'all', [switch]$Diagnostic) $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' $WarningPreference = 'SilentlyContinue' $InformationPreference = 'SilentlyContinue' $env:PSModulePath = Join-Path $PSHOME 'Modules' +if ($PSVersionTable.PSEdition -eq 'Core') { + # Inbox Windows modules remain under WindowsPowerShell when invoked from PS7. + $env:PSModulePath += ';' + (Join-Path ([Environment]::SystemDirectory) 'WindowsPowerShell\v1.0\Modules') +} +$script:SamsaraDiagnostic = $Diagnostic.IsPresent [Console]::OutputEncoding = New-Object System.Text.UTF8Encoding($false) if ([Environment]::OSVersion.Platform -ne [PlatformID]::Win32NT) { [Console]::Error.WriteLine('This collector requires a local Windows Server execution host.') @@ -30,7 +35,7 @@ $collectors = [ordered]@{ } if ($Section -ne 'all') { $result = Invoke-SamsaraSection $Section $collectors[$Section] - ConvertTo-Json -InputObject $result -Depth 12 -Compress + ConvertTo-SamsaraJson $result 12 exit 0 } # Direct PowerShell use emits the same complete inventory envelope as the Python backend. @@ -50,7 +55,7 @@ foreach ($name in $collectors.Keys) { } elseif ($name -eq 'shares') { $inventory.storage += @($result.data) } else { $inventory[$name] = @($result.data) } } -ConvertTo-Json -InputObject ([ordered]@{ schema_version='1.0'; synthetic=$false; collected_at=$collectedAt - server=$server; checks=$checks; inventory=$inventory }) -Depth 14 -Compress +ConvertTo-SamsaraJson ([ordered]@{ schema_version='1.0'; synthetic=$false; collected_at=$collectedAt + server=$server; checks=$checks; inventory=$inventory }) 14 exit 0 diff --git a/powershell/Get-ServerTasks.ps1 b/powershell/Get-ServerTasks.ps1 index 796576d..733fc6b 100644 --- a/powershell/Get-ServerTasks.ps1 +++ b/powershell/Get-ServerTasks.ps1 @@ -1,8 +1,10 @@ function Get-SamsaraTasks { param($Result) Invoke-SamsaraRead $Result 'Scheduled tasks' { - $Result.data = @(Get-ScheduledTask -ErrorAction Stop | ForEach-Object { - $task = $_ + $tasks = @(Get-ScheduledTask -ErrorAction Stop) + foreach ($task in $tasks) { + Invoke-SamsaraRead $Result 'Scheduled task metadata projection' { + if ($null -eq $task -or [string]::IsNullOrEmpty([string]$task.TaskName)) { throw 'Missing task identity' } $triggers = @($task.Triggers | ForEach-Object { $type = if ($_.CimClass) { [string]$_.CimClass.CimClassName } else { 'Unknown trigger' } # Select only known scheduler metadata; never serialize entire CIM objects. @@ -10,11 +12,14 @@ function Get-SamsaraTasks { '{0}; start={1}; enabled={2}; daysInterval={3}; weeksInterval={4}; daysOfWeek={5}; repeat={6}' -f $type, $_.StartBoundary, $_.Enabled, $_.DaysInterval, $_.WeeksInterval, $_.DaysOfWeek, $repetition }) $executables = @($task.Actions | ForEach-Object { Get-SamsaraSafeExecutable ([string]$_.Execute) }) - [pscustomobject]@{ name = [string]$task.TaskName; path = [string]$task.TaskPath + $row = [pscustomobject]@{ name = [string]$task.TaskName; path = [string]$task.TaskPath enabled = [bool]$task.Settings.Enabled; state = [string]$task.State - account = [string]$task.Principal.UserId; executables = ($executables -join '; ') + account = if ($task.Principal.UserId) { [string]$task.Principal.UserId } else { [string]$task.Principal.GroupId }; executables = ($executables -join '; ') triggers = if ($triggers.Count) { $triggers -join ' | ' } else { 'No trigger metadata visible; manual or external invocation remains possible' } } - }) + $null = ConvertTo-Json -InputObject $row -Depth 4 -ErrorAction Stop + $Result.data += $row + } + } } $Result.limitations += 'Visible Task Scheduler registrations only. Arguments, inline scripts, task XML, event queries and execution history are omitted. Non-executable actions may lack an executable name. Trigger fields are descriptive, not a prediction of execution.' } diff --git a/powershell/Inventory.Common.ps1 b/powershell/Inventory.Common.ps1 index 765bb02..8bab72e 100644 --- a/powershell/Inventory.Common.ps1 +++ b/powershell/Inventory.Common.ps1 @@ -13,6 +13,13 @@ function Add-SamsaraError { 'command_unavailable' { 'Required local inventory command is unavailable.' } default { 'Local query could not complete; raw diagnostics are excluded.' } } + if ($script:SamsaraDiagnostic) { + # Fixed stages only; never include messages, identifiers, targets or stacks. + $type = $Record.Exception.GetType().Name + if ($type -notmatch '^[A-Za-z0-9]+Exception$') { $type = 'Exception' } + $category = [string]$Record.CategoryInfo.Category + $message += ' Exception type: ' + $type + '; category: ' + $category + '.' + } $Result.errors += [pscustomobject]@{ code = $code; scope = $Scope; message = $message } if ($code -eq 'access_denied') { $Result.permission_limitations += $Scope + ': read access denied.' } } @@ -21,15 +28,28 @@ function Invoke-SamsaraRead { param($Result, [string]$Scope, [scriptblock]$Query) $Result.attempted_subchecks++ try { + $ErrorActionPreference = 'Stop' & $Query $Result.successful_subchecks++ } catch { Add-SamsaraError $Result $_ $Scope } } +function ConvertTo-SamsaraJson { + param($Value, [int]$Depth = 14) + try { ConvertTo-Json -InputObject $Value -Depth $Depth -Compress -ErrorAction Stop } + catch { + # A serialization failure must not emit a complete empty inventory. + [Console]::Error.WriteLine('Inventory JSON serialization failed; raw diagnostics are excluded.') + exit 3 + } +} + function Get-SamsaraSafeExecutable { param([string]$Value) # Task action arguments, scripts and environment values are deliberately omitted. - $name = [IO.Path]::GetFileName(($Value -replace '\\', '/')) + # Scheduler action values need not be valid filesystem paths. Path.GetFileName + # throws on malformed/quoted values in .NET Framework (PowerShell 5.1). + $name = (($Value -replace '\\', '/') -split '/')[-1] if ($name -match '^[A-Za-z0-9_. -]+\.(exe|com|cmd|bat|ps1|vbs|js|wsf|msc)$') { return $name } return '[executable name omitted]' } diff --git a/samsara/collectors/windows.py b/samsara/collectors/windows.py index 8915bad..e22fa5b 100644 --- a/samsara/collectors/windows.py +++ b/samsara/collectors/windows.py @@ -48,12 +48,14 @@ def powershell_executable(): return executable -def run_section(section, executable, cancelled=lambda: False, timeout=45): +def run_section(section, executable, cancelled=lambda: False, timeout=45, diagnostic=False): if section not in SECTIONS: raise ValueError('Unknown Windows inventory section.') if cancelled(): raise CollectionCancelled() args = [str(executable), '-NoLogo', '-NoProfile', '-NonInteractive', '-File', str(SCRIPT), '-Section', section] + if diagnostic: + args.append('-Diagnostic') environment = os.environ.copy() # Only inbox modules are needed. Do not autoload user-supplied modules or profiles. environment['PSModulePath'] = str(Path(executable).parent / 'Modules') @@ -77,6 +79,8 @@ def run_section(section, executable, cancelled=lambda: False, timeout=45): if cancelled(): raise CollectionCancelled() if process.returncode: + if process.returncode == 3: + raise WindowsCollectionError('serialization_failed', 'PowerShell could not serialize inventory metadata; no successful evidence was substituted.') raise WindowsCollectionError('powershell_failed', 'The trusted script could not run. Check script signing policy, module availability and read permissions; raw diagnostics are excluded.') output.seek(0) raw = output.read(MAX_OUTPUT + 1) @@ -127,7 +131,7 @@ def failure_check(code, message, section, start, status='failed'): def collect_local_windows(stage=lambda message: None, cancelled=lambda: False, profile='auto', - section_timeout=45, total_timeout=300, runner=None): + section_timeout=45, total_timeout=300, runner=None, diagnostic=False): if platform.system() != 'Windows': raise ValueError('Local Windows collection requires an authorized Windows Server execution host.') if profile != 'auto' and not profile.startswith('windows:'): @@ -150,7 +154,10 @@ def collect_local_windows(stage=lambda message: None, cancelled=lambda: False, p raise ValueError('The Windows identity check could not run within the time limit.') continue try: - record = validate_section(runner(section, executable, cancelled=cancelled, timeout=min(section_timeout, remaining)), section) + options = {'cancelled': cancelled, 'timeout': min(section_timeout, remaining)} + if diagnostic: + options['diagnostic'] = True + record = validate_section(runner(section, executable, **options), section) total_bytes += len(json.dumps(record).encode('utf-8')) if total_bytes > MAX_OUTPUT: raise WindowsCollectionError('output_limit', 'Total inventory exceeded the 10 MiB limit; this section was omitted.') @@ -178,10 +185,11 @@ def collect_local_windows(stage=lambda message: None, cancelled=lambda: False, p def main(): parser = argparse.ArgumentParser(description='Read-only local Windows Server inventory. No remoting is used.') parser.add_argument('--output', required=True, help='New local JSON file to create') + parser.add_argument('--diagnostic', action='store_true', help='Include sanitized exception types and categories') parser.add_argument('--profile', choices=['auto','windows:2016','windows:2019','windows:2022','windows:2025'], default='auto') args = parser.parse_args() try: - data = collect_local_windows(profile=args.profile) + data = collect_local_windows(profile=args.profile, diagnostic=args.diagnostic) with open(args.output, 'x', encoding='utf-8') as output: json.dump(data, output, indent=2) print('Windows inventory saved. Review all failed/partial checks and permission limitations before retirement decisions.') diff --git a/samsara/ui/window.py b/samsara/ui/window.py index 29d8193..11c7cb1 100644 --- a/samsara/ui/window.py +++ b/samsara/ui/window.py @@ -476,6 +476,8 @@ class MainWindow(QMainWindow): if key in self.data['checks']: check = self.data['checks'][key] area.addWidget(self.label(f"{check['status'].replace('_', ' ').title()}: {check['message']}", muted=True)) + for error in check.get('errors', []): + area.addWidget(self.label(f"{error['scope']} [{error['code']}]: {error['message']}", muted=True)) area.addWidget(SearchTable(rows)) def job_finished(self): diff --git a/tests/powershell/Test-Collectors.ps1 b/tests/powershell/Test-Collectors.ps1 index 1291bef..598dbc3 100644 --- a/tests/powershell/Test-Collectors.ps1 +++ b/tests/powershell/Test-Collectors.ps1 @@ -82,3 +82,63 @@ $script:SamsaraIsElevated=$false $section=Invoke-SamsaraSection 'services' {param($r) Get-SamsaraServices $r} Assert-True ($section.status -eq 'partial' -and $section.permission_limitations.Count -gt 0) 'Non-elevated scope not marked partial' Write-Output 'PowerShell AST and synthetic collector tests passed. No real Windows inventory was collected.' +# Reliability cases: mocks never access real tasks, certificates or private keys. +$script:SamsaraIsElevated=$true +$script:SamsaraDiagnostic=$true +function New-TestTask { + param([bool]$Enabled=$true) + [pscustomobject]@{TaskName='Example';TaskPath='\';State='Ready';Settings=[pscustomobject]@{Enabled=$Enabled};Principal=[pscustomobject]@{UserId='SYSTEM'};Actions=@([pscustomobject]@{Execute='"C:\Tools\bad.exe"';Arguments='SECRET'});Triggers=@()} +} +function Get-ScheduledTask { [CmdletBinding()]param() New-TestTask $true; New-TestTask $false } +$r=Invoke-SamsaraSection tasks {param($r) Get-SamsaraTasks $r} +Assert-True ($r.status -eq 'complete' -and $r.data.Count -eq 2 -and -not $r.data[1].enabled) 'Normal/disabled tasks lost' +Assert-True ($r.data[0].executables -eq '[executable name omitted]') 'Malformed executable not sanitized' +function Get-ScheduledTask { [CmdletBinding()]param() } +$r=Invoke-SamsaraSection tasks {param($r) Get-SamsaraTasks $r} +Assert-True ($r.status -eq 'complete' -and $r.data.Count -eq 0) 'Successful empty task query failed' +function Get-ScheduledTask { [CmdletBinding()]param() Write-Error 'SECRET' -Category PermissionDenied } +$r=Invoke-SamsaraSection tasks {param($r) Get-SamsaraTasks $r} +Assert-True ($r.status -eq 'failed' -and $r.data.Count -eq 0) 'Failed task query treated as empty success' +function Get-ScheduledTask { + [CmdletBinding()]param() + New-TestTask + $bad=New-TestTask + $bad | Add-Member ScriptProperty TaskName { throw [UnauthorizedAccessException]::new('SECRET') } -Force + # Throw from ToString to reliably exercise projection failures on both runtimes. + $value=New-Object PSObject + $value | Add-Member ScriptMethod ToString { throw [UnauthorizedAccessException]::new('SECRET') } -Force + $bad.TaskPath=$value + $bad +} +$r=Invoke-SamsaraSection tasks {param($r) Get-SamsaraTasks $r} +Assert-True ($r.status -eq 'partial' -and $r.data.Count -eq 1 -and $r.errors.Count -gt 0) 'Individual task failure discarded healthy records' +Assert-True (($r | ConvertTo-Json -Depth 12) -notmatch 'SECRET') 'Diagnostic leaked secret' +$script:StoreMode='normal' +function Get-ChildItem { + [CmdletBinding()]param($LiteralPath) + if ($LiteralPath -eq 'Cert:\LocalMachine') { + foreach ($n in @('My','Root','Empty')) { [pscustomobject]@{PSChildName=$n;PSPath=('Microsoft.PowerShell.Security\Certificate::'+$n)} } + } elseif ($LiteralPath -eq 'Cert:\LocalMachine\Empty') { return } + elseif ($LiteralPath -in @('Cert:\LocalMachine\My','Cert:\LocalMachine\Root')) { + if ($script:StoreMode -eq 'denied' -or ($script:StoreMode -eq 'partial' -and $LiteralPath -like '*Root')) { throw [UnauthorizedAccessException]::new('SECRET') } + [pscustomobject]@{Subject='CN=Example';Issuer='CN=CA';Thumbprint='00';NotAfter=[datetime]::UtcNow;PrivateKey='SECRET'} + } else { throw 'Incorrect certificate provider path' } +} +$r=Invoke-SamsaraSection certificates {param($r) Get-SamsaraCertificates $r} +Assert-True ($r.status -eq 'complete' -and $r.data.Count -eq 2 -and @($r.data.store | Select-Object -Unique).Count -eq 2) 'Multiple stores/empty store failed' +$script:StoreMode='partial' +$r=Invoke-SamsaraSection certificates {param($r) Get-SamsaraCertificates $r} +Assert-True ($r.status -eq 'partial' -and $r.data.Count -eq 1 -and $r.errors[0].code -eq 'access_denied') 'Partial store failure lost records' +Assert-True (($r | ConvertTo-Json -Depth 12) -notmatch 'SECRET') 'Certificate diagnostics leaked secret' +function Get-ChildItem { [CmdletBinding()]param($LiteralPath) if ($LiteralPath -eq 'Cert:\LocalMachine') { [pscustomobject]@{PSChildName='My'} } else { throw [UnauthorizedAccessException]::new('SECRET') } } +$r=Invoke-SamsaraSection certificates {param($r) Get-SamsaraCertificates $r} +Assert-True ($r.status -eq 'failed' -and $r.data.Count -eq 0) 'Inaccessible stores treated as empty success' +function Get-ChildItem { [CmdletBinding()]param($LiteralPath) } +$r=Invoke-SamsaraSection certificates {param($r) Get-SamsaraCertificates $r} +Assert-True ($r.status -eq 'complete' -and $r.data.Count -eq 0) 'Empty store list failed' +function Get-ScheduledTask { [CmdletBinding()]param() New-TestTask } +function ConvertTo-Json { [CmdletBinding()]param($InputObject,$Depth,[switch]$Compress) throw [InvalidOperationException]::new('SECRET serialization') } +$r=Invoke-SamsaraSection tasks {param($r) Get-SamsaraTasks $r} +Assert-True ($r.status -ne 'complete' -and $r.data.Count -eq 0 -and $r.errors.Count -eq 1) 'Serialization failure accepted as evidence' +Remove-Item Function:\ConvertTo-Json +Write-Output 'Reliability collector tests passed.' diff --git a/tests/test_powershell.py b/tests/test_powershell.py index c2a5c67..2f1d9b9 100644 --- a/tests/test_powershell.py +++ b/tests/test_powershell.py @@ -8,6 +8,12 @@ import unittest class PowerShellTests(unittest.TestCase): def test_native_ast_and_synthetic_collectors(self): executable = shutil.which('powershell.exe') if os.name == 'nt' else shutil.which('pwsh') + self.run_collectors(executable) + + def test_powershell_7_collectors(self): + self.run_collectors(shutil.which('pwsh')) + + def run_collectors(self, executable): if not executable: self.skipTest('No PowerShell runtime available; native AST/collector tests require Windows PowerShell 5.1 or pwsh.') script=Path('tests/powershell/Test-Collectors.ps1').resolve() diff --git a/tests/test_scoring.py b/tests/test_scoring.py index b4941c4..2e17297 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -20,6 +20,17 @@ def scored(result): class ScoringRegressionTests(unittest.TestCase): + def test_task_and_certificate_failures_reduce_coverage_without_risk(self): + data = fixture('clean_windows') + baseline = assess(data) + for name, status in [('tasks', 'failed'), ('certificates', 'partial')]: + data['checks'][name]['status'] = status + data['checks'][name]['errors'] = [{'code': 'collection_failed', 'scope': name, 'message': 'Sanitized failure'}] + result = assess(data) + self.assertEqual(result['score'], baseline['score']) + self.assertLess(result['coverage']['percent'], baseline['coverage']['percent']) + self.assertEqual(result['confidence']['level'], 'moderate') + def test_required_scenarios(self): expected = {'clean_windows':0,'inactive_game_server':0,'active_sql_server':70, 'active_iis_server':70,'domain_controller':95,'active_file_server':80, diff --git a/tests/test_windows.py b/tests/test_windows.py index 8836471..f4b0caa 100644 --- a/tests/test_windows.py +++ b/tests/test_windows.py @@ -155,6 +155,19 @@ class FakeProcess: class ProcessTests(unittest.TestCase): + def test_serialization_exit_and_diagnostic_switch(self): + def create(args, **kwargs): + self.assertIn('-Diagnostic', args) + process = FakeProcess(args, **kwargs) + process.returncode = 3 + kwargs['stdout'].write(b'SECRET') + return process + with patch('samsara.collectors.windows.subprocess.Popen', side_effect=create): + with self.assertRaises(WindowsCollectionError) as error: + run_section('tasks', 'powershell.exe', diagnostic=True) + self.assertEqual(error.exception.code, 'serialization_failed') + self.assertNotIn('SECRET', str(error.exception)) + def test_timeout_and_cancel_kill_and_reap_without_shell(self): for cancel in [False,True]: processes=[]