Replace inventory-counting risk with bounded workload scoring and separate confidence

tests/powershell/Test-Collectors.ps1

@@ -33,6 +33,11 @@ function Get-ScheduledTask {
 }
 function Get-Volume { [CmdletBinding()] param() [pscustomobject]@{ FileSystemLabel='Synthetic OS'; DriveLetter='C'; Path='C:\'; FileSystem='NTFS'; Size=100GB; SizeRemaining=50GB } }
 function Get-SmbShare { [CmdletBinding()] param() [pscustomobject]@{ Name='SyntheticReports'; Path='C:\Reports'; Special=$false } }
+function Get-SmbOpenFile {
+    [CmdletBinding()] param()
+    [pscustomobject]@{ Path='C:\Reports\SECRET-file.txt'; ShareRelativePath='SECRET-file.txt'; ClientComputerName='192.0.2.21'; ClientUserName='SECRET' }
+    [pscustomobject]@{ Path='C:\Reports\SECRET-admin.txt'; ShareRelativePath='Reports\SECRET-admin.txt'; ClientComputerName='192.0.2.21'; ClientUserName='SECRET' }
+}
 function Get-ChildItem {
     [CmdletBinding()] param($LiteralPath)
     if ($LiteralPath -eq 'Cert:\LocalMachine') { [pscustomobject]@{ PSPath='Cert:\LocalMachine\My'; PSChildName='My' } }
@@ -51,6 +56,7 @@ foreach ($name in $collectors.Keys) {
     Assert-True ($section.started_at -and $section.completed_at) 'Missing timestamps'
     Assert-True (($section | ConvertTo-Json -Depth 12) -notmatch 'SECRET') 'Secret field serialized'
     if ($name -eq 'network') { Assert-True ($section.data[0].process -eq 'SyntheticWorker') 'Process correlation failed' }
+    if ($name -eq 'shares') { Assert-True ($section.data[0].observed_open_files -eq 1) 'Share activity correlation missing' }
     if ($name -eq 'tasks') { Assert-True ($section.data[0].executables -eq 'report.exe') 'Executable basename missing' }
 }
 foreach ($build in @('14393','17763','20348','26100')) {
@@ -69,7 +75,7 @@ $section=Invoke-SamsaraSection 'software' {param($r) Invoke-SamsaraRead $r 'Synt
 Assert-True ($section.status -eq 'complete' -and $section.data[0].name -eq 'Synthetic application') 'Registry metadata projection failed'
 function Get-SmbShare { [CmdletBinding()] param() throw [UnauthorizedAccessException]::new('SECRET protected details') }
 $section=Invoke-SamsaraSection 'shares' {param($r) Get-SamsaraShares $r}
-Assert-True ($section.status -eq 'failed') 'Denied collection was not failed'
+Assert-True ($section.status -eq 'partial' -and $section.data.Count -eq 0) 'Denied share enumeration was not partial/empty'
 Assert-True ($section.errors[0].code -eq 'access_denied') 'Permission error not categorized'
 Assert-True (($section | ConvertTo-Json -Depth 12) -notmatch 'SECRET') 'Raw error exposed'
 $script:SamsaraIsElevated=$false

tests/test_core.py

@@ -25,7 +25,8 @@ class CoreTests(unittest.TestCase):
         data = collect_demo(DEMO_HOST)
         result = assess(data)
         self.assertTrue(all(f['evidence'] for f in result['findings']))
-        self.assertTrue(any(f['classification'] == 'Confirmed observed activity' for f in result['findings']))
+        self.assertTrue(any(f['classification'] == 'Potential workload dependency' for f in result['findings']))
+        self.assertFalse(any(f['classification'] == 'Confirmed workload dependency' for f in result['findings']))
         data['inventory']['network'] = []
         self.assertTrue(any('does not prove' in f['evidence'] for f in assess(data)['findings']))
         self.assertTrue(any('ad:' in f['evidence'] for f in result['findings']))

tests/test_scoring.py

@@ -0,0 +1,215 @@
+import copy
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from samsara.analyzers.assessment import assess
+from samsara.analyzers.dependencies import CONFIRMED, POTENTIAL, STANDARD, UNKNOWN
+from samsara.analyzers.policy import load_policy
+from samsara.models.inventory import validate_inventory
+from samsara.reporting.export import export_html, export_json
+from samsara.collectors.importer import load_inventory
+
+
+def fixture(name):
+    return json.loads(Path('tests/fixtures/scoring',name+'.json').read_text())
+
+
+def scored(result):
+    return [row for row in result['findings'] if row['points']>0]
+
+
+class ScoringRegressionTests(unittest.TestCase):
+    def test_required_scenarios(self):
+        expected = {'clean_windows':0,'inactive_game_server':0,'active_sql_server':70,
+                    'active_iis_server':70,'domain_controller':95,'active_file_server':80,
+                    'incomplete_collection':0}
+        for name,score in expected.items():
+            with self.subTest(name=name):
+                data=fixture(name)
+                validate_inventory(data)
+                before=copy.deepcopy(data)
+                result=assess(data)
+                self.assertEqual(result['score'],score)
+                self.assertEqual(data,before,'Analyzer mutated the raw inventory')
+                self.assertTrue(result['review_required'])
+                self.assertIn('Historical usage validation',result['manual_verification'])
+                self.assertTrue(all(f['evidence'] and f['scoring_reason'] for f in result['findings']))
+                self.assertEqual(result['score'],max((r['score'] for r in result['scoring']['breakdown']),default=0))
+                self.assertTrue(all(0<=f['points']<=100 for f in result['findings']))
+
+    def test_original_1565_point_incident_is_not_a_workload(self):
+        data=fixture('incomplete_collection')
+        inventory=data['inventory']
+        self.assertEqual(len(inventory['services']),86)
+        self.assertEqual(len(inventory['roles']),14)
+        self.assertEqual(sum(r['state']=='Established' for r in inventory['network']),19)
+        self.assertEqual(sum(r['state']=='Listen' for r in inventory['network']),23)
+        self.assertEqual(len(inventory['storage']),3)
+        old_points=86*10+19*20+14*10+23*5+3*15+2*10+5
+        self.assertEqual(old_points,1565)
+        result=assess(data)
+        self.assertEqual(result['score'],0)
+        self.assertFalse(scored(result))
+        self.assertLess(len(result['findings']),15)
+        self.assertLess(result['coverage']['percent'],100)
+
+    def test_standard_os_services_features_and_admin_shares_stay_visible(self):
+        data=fixture('clean_windows')
+        result=assess(data)
+        standard=[r for r in result['findings'] if r['classification']==STANDARD]
+        self.assertTrue(standard)
+        self.assertTrue(all(r['points']==0 for r in standard))
+        for share in data['inventory']['storage']:
+            share.update(observed_open_files=100,activity_source='Get-SmbOpenFile (share-root-correlated)')
+        self.assertEqual(assess(data)['score'],0)
+        # A hidden business share is not a default administrative share.
+        data['inventory']['storage'].append({'type':'SMB Share','name':'Finance$','path':'D:\\Finance','special':True})
+        self.assertEqual(assess(data)['score'],20)
+
+    def test_running_service_listener_and_connections_are_one_workload(self):
+        data=fixture('active_sql_server')
+        baseline=assess(data)
+        self.assertEqual(len(scored(baseline)),1)
+        self.assertEqual(scored(baseline)[0]['id'],'sql')
+        self.assertEqual(scored(baseline)[0]['classification'],CONFIRMED)
+        data['inventory']['network'] *= 20
+        data['inventory']['services'] *= 10
+        repeated=assess(data)
+        self.assertEqual(repeated['score'],70)
+        self.assertEqual(len(scored(repeated)),1)
+        selected=repeated['scoring']['breakdown'][0]
+        self.assertIn('observed_application_clients',[r['selected_rule'] for r in repeated['scoring']['breakdown']])
+        self.assertTrue(selected['selected_evidence_refs'])
+        self.assertIn('Observed application TCP activity',scored(repeated)[0]['evidence'])
+
+    def test_independent_workloads_use_maximum_not_sum(self):
+        data=fixture('active_sql_server')
+        other=fixture('active_file_server')['inventory']['storage'][-1]
+        data['inventory']['storage'].append(other)
+        result=assess(data)
+        self.assertEqual(sorted(r['score'] for r in result['scoring']['breakdown']),[70,80])
+        self.assertEqual(result['score'],80)
+        self.assertEqual(sum(r['determines_final_score'] for r in result['scoring']['breakdown']),1)
+
+    def test_coverage_does_not_inflate_or_mask_known_risk(self):
+        clean=assess(fixture('clean_windows'))
+        incomplete=assess(fixture('incomplete_collection'))
+        self.assertEqual(clean['score'],incomplete['score'])
+        self.assertLess(incomplete['coverage']['percent'],clean['coverage']['percent'])
+        self.assertEqual(clean['confidence']['level'],'high')
+        self.assertEqual(incomplete['confidence']['level'],'moderate')
+        data=fixture('active_sql_server')
+        for check in data['checks'].values():
+            check['status']='failed'
+        result=assess(data)
+        self.assertEqual(result['score'],70,'Missing scope cannot erase actual captured positive evidence')
+        self.assertEqual(result['confidence']['level'],'low')
+        self.assertEqual(result['coverage']['percent'],0)
+        self.assertTrue(all(f['points']==0 for f in result['findings'] if f['category']=='Missing evidence'))
+
+    def test_browser_steam_loopback_outbound_and_unknown_are_not_inbound(self):
+        self.assertFalse(any(r['id']=='game-server' for r in assess(fixture('inactive_game_server'))['findings']))
+        data=fixture('active_sql_server')
+        # Remove active external connections, keep the running service/listener.
+        row=data['inventory']['network'][-1]
+        row['remote_address']='127.0.0.1'
+        result=assess(data)
+        self.assertEqual(result['score'],45)
+        self.assertEqual(result['network_observations'][-1]['direction'],'Loopback / same host')
+        self.assertFalse(any(f['classification']==CONFIRMED for f in scored(result)))
+        row['remote_address']='::ffff:127.0.0.1'
+        self.assertEqual(assess(data)['network_observations'][-1]['direction'],'Loopback / same host')
+        row.update(remote_address='192.0.2.50',direction='Outbound')
+        self.assertEqual(assess(data)['score'],45)
+        row.update(pid=99999,direction='Unknown',local_port=1433)
+        self.assertEqual(assess(data)['network_observations'][-1]['direction'],'Unknown')
+        self.assertEqual(assess(data)['score'],45)
+        row.update(local_port=55200,remote_port=443)
+        self.assertEqual(assess(data)['network_observations'][-1]['direction'],'Outbound (inferred)')
+
+    def test_port_alone_and_iis_http_sys_are_not_proof_of_a_hosted_app(self):
+        data=fixture('clean_windows')
+        data['inventory']['network'].append({'state':'Listen','local_address':'0.0.0.0','local_port':27015,'pid':12345,'process':'unknown','remote_address':'','remote_port':0,'direction':'Unknown'})
+        self.assertEqual(assess(data)['score'],0)
+        data=fixture('active_iis_server')
+        for row in data['inventory']['network'][-2:]:
+            row.update(pid=4,process='System')
+        result=assess(data)
+        self.assertEqual(result['score'],35)
+        self.assertFalse(any(f['classification']==CONFIRMED for f in scored(result)))
+
+    def test_role_only_not_domain_controller_and_smb_transport_not_share_use(self):
+        data=fixture('domain_controller');data['server']['product_type']=3
+        data['inventory']['services']=[r for r in data['inventory']['services'] if r['name']!='NTDS']
+        self.assertEqual(assess(data)['score'],15)
+        data=fixture('active_file_server')
+        share=data['inventory']['storage'][-1]
+        share['activity_source']='Get-SmbOpenFile (administrative alias ambiguous)'
+        self.assertEqual(assess(data)['score'],40)
+        share.pop('observed_open_files')
+        result=assess(data)
+        self.assertEqual(result['score'],40)
+        self.assertEqual(scored(result)[0]['classification'],POTENTIAL)
+        self.assertIn('accessed share is unknown',scored(result)[0]['evidence'])
+
+    def test_policy_validation_configuration_and_audit_snapshot(self):
+        policy=load_policy()
+        policy['scores']['observed_application_clients']=55
+        result=assess(fixture('active_sql_server'),policy=policy)
+        self.assertEqual(result['score'],55)
+        self.assertEqual(result['scoring']['policy']['scores']['observed_application_clients'],55)
+        policy['scores']['observed_application_clients']=2
+        self.assertEqual(result['scoring']['policy']['scores']['observed_application_clients'],55)
+        for source in [[],{},dict(load_policy(),model_version='1.0')]:
+            with self.subTest(source=type(source)),self.assertRaises(ValueError):
+                load_policy(source)
+        bad=load_policy();bad['scores']['running_application_service']=-1
+        with self.assertRaises(ValueError): load_policy(bad)
+        bad=load_policy();bad['thresholds']['medium']=90
+        with self.assertRaises(ValueError): load_policy(bad)
+
+    def test_actionable_html_and_separate_complete_inventory(self):
+        data=fixture('clean_windows')
+        with tempfile.TemporaryDirectory() as folder:
+            report=Path(folder)/'assessment.html'
+            export_html(report,data,assess(data))
+            text=report.read_text()
+            self.assertIn('How the score was calculated',text)
+            self.assertIn('Evidence confidence',text)
+            self.assertIn('Historical usage validation',text)
+            self.assertNotIn('medium-risk dependencies',text)
+            self.assertLess(text.count('<tr>'),60)
+            inventory=list(Path(folder).glob('*.inventory-*.json'))
+            self.assertEqual(len(inventory),1)
+            raw=json.loads(inventory[0].read_text())
+            self.assertEqual(raw,data)
+            self.assertEqual(len(raw['inventory']['services']),86)
+            # Re-export reuses an identical artifact rather than losing inventory.
+            export_html(report,data,assess(data))
+            export_json(Path(folder)/'assessment.json',data,assess(data))
+            self.assertEqual(load_inventory(inventory[0])['inventory'],load_inventory(Path(folder)/'assessment.json')['inventory'])
+
+    def test_custom_jobs_not_misclassified_by_substrings_and_zero_points_maintenance(self):
+        data=fixture('clean_windows')
+        data['inventory']['tasks'].append({'name':'capture-report','path':'/etc/cron.d/custom','enabled':True,'triggers':'0 2 * * *'})
+        self.assertEqual(assess(data)['score'],20)
+        data['inventory']['tasks'][-1]['enabled']=False
+        self.assertEqual(assess(data)['score'],0)
+        data['inventory']['tasks'][-1].update(name='Business export',path='\\Microsoft\\Windows\\PowerShell\\ScheduledJobs\\',enabled=True)
+        self.assertEqual(assess(data)['score'],20)
+
+    def test_html_escapes_actionable_evidence_and_artifacts_are_collision_safe(self):
+        data=fixture('active_sql_server')
+        data['inventory']['services'][-1]['account']='<script>owned</script>'
+        result=assess(data)
+        with tempfile.TemporaryDirectory() as folder:
+            report=Path(folder)/'assessment.html'
+            export_html(report,data,result)
+            self.assertNotIn('<script>',report.read_text())
+            self.assertIn('&lt;script&gt;',report.read_text())
+            artifact=next(Path(folder).glob('*.inventory-*.json'))
+            artifact.write_text('unrelated data')
+            with self.assertRaises(FileExistsError):
+                export_html(report,data,result)
+            self.assertEqual(artifact.read_text(),'unrelated data')

tests/test_ui.py

@@ -141,3 +141,33 @@ class UITests(unittest.TestCase):
                     window.quick_save.click()
                 self.assertIn('LOCAL WINDOWS INVENTORY', report.read_text())
             window.close()
+
+    def test_policy_changes_preserve_history_and_confidence_is_visible(self):
+        import json
+        from samsara.analyzers.policy import load_policy
+        window=MainWindow()
+        fixture=Path('tests/fixtures/scoring/active_sql_server.json')
+        window.collection_done(json.loads(fixture.read_text()))
+        self.assertEqual(window.assessment['score'],70)
+        original_policy=window.assessment['scoring']['policy_id']
+        self.assertIn('Confidence:',window.coverage_note.text())
+        self.assertIn('Coverage gaps do not add risk points',window.coverage_note.text())
+        with tempfile.TemporaryDirectory() as folder:
+            policy=load_policy()
+            policy['scores']['observed_application_clients']=55
+            path=Path(folder)/'policy.json'
+            path.write_text(json.dumps(policy))
+            with patch('samsara.ui.window.QFileDialog.getOpenFileName',return_value=(str(path),'')):
+                window.choose_policy()
+            window.source.setCurrentIndex(window.source.findData('file'))
+            window.inventory_path.setText(str(fixture.resolve()))
+            window.start_collection()
+            self.assertFalse(window.policy_button.isEnabled())
+            self.finish(window)
+            self.assertEqual(window.assessment['score'],55)
+            self.assertNotEqual(window.assessment['scoring']['policy_id'],original_policy)
+            window.open_history(window.history_list.item(0))
+            self.assertEqual(window.assessment['score'],70)
+            self.assertEqual(window.assessment['scoring']['policy_id'],original_policy)
+            window.reset_policy()
+        window.close()