Privacy hardening, remote-peer IP masking and review fixes

- Add privacy projection (core/privacy.py), private report storage, bounded
  process capture; omit identities, free text and unapproved paths
- Mask remote peer IPs: only loopback/host/wildcard stay numeric, others are
  reduced to remote_scope private/public (schema + scoring updated)
- Map ss TCP states to canonical names; ASCII-only numeric fields; fix
  "Microsoft Visual C++" product matching (Python and PowerShell)
- Merge 7fed3b9: -Diagnostic switch, serialization exit code 3, PS7 module
  path, UI error detail; keep hardened task/certificate collectors
- Add audit docs, privacy docs and regression tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-04 18:50:24 -08:00
parent 7fed3b9ab9
commit 610a3bb447
51 changed files with 2719 additions and 604 deletions

View File

@@ -1,53 +1,31 @@
"""Offline intake shared by Windows and Linux desktops. No code is executed."""
import json
import os
import stat
from pathlib import Path
from samsara.models.inventory import validate_inventory, restrict_inventory_fields
from samsara.core.privacy import minimize_inventory
from samsara.core.storage import require_local_path
from samsara.models.inventory import validate_inventory
MAX_IMPORT_BYTES = 10 * 1024 * 1024
def load_inventory(path):
with Path(path).open('rb') as source:
local = require_local_path(path)
flags=os.O_RDONLY|getattr(os,'O_NOFOLLOW',0)|getattr(os,'O_NONBLOCK',0)|getattr(os,'O_BINARY',0)
with os.fdopen(os.open(local,flags),'rb') as source:
if not stat.S_ISREG(os.fstat(source.fileno()).st_mode):
raise ValueError('Only regular local inventory files may be imported.')
raw = source.read(MAX_IMPORT_BYTES + 1)
if len(raw) > MAX_IMPORT_BYTES:
raise ValueError('Inventory exceeds the 10 MiB import limit.')
try:
data = json.loads(raw.decode('utf-8-sig'))
except (ValueError, UnicodeError):
except (ValueError, UnicodeError, RecursionError):
raise ValueError('The selected file is not valid UTF-8 inventory JSON.') from None
# A report wrapper is accepted, but its precomputed assessment is never trusted.
if isinstance(data, dict) and 'inventory_result' in data:
data = data['inventory_result']
validate_inventory(data)
# Keep diagnostic contents out of logs and exported reports. Diagnostics may
# contain command lines or credentials from external collection tools.
safe_errors = {
'access_denied': 'Current account could not read this scope.',
'command_unavailable': 'Required command was unavailable.',
'timeout': 'Section time limit exceeded.',
'overall_timeout': 'Overall collection time limit exceeded.',
'output_limit': 'Collection output limit exceeded.',
'powershell_failed': 'PowerShell could not execute the collector.',
'invalid_output': 'Invalid section output.',
'collection_failed': 'A local query failed.'
}
checks = {}
for name, check in data['checks'].items():
clean = {'status': check['status'], 'message':
'Imported check status. Source assertions are unverified; raw diagnostics excluded.'}
for field in ('started_at', 'completed_at'):
if field in check:
clean[field] = check[field]
if 'errors' in check:
clean['errors'] = []
for error in check['errors']:
code = error['code'] if error['code'] in safe_errors else 'collection_failed'
clean['errors'].append({'code': code, 'scope': 'Imported check', 'message': safe_errors[code]})
if 'permission_limitations' in check:
clean['permission_limitations'] = ['Collector reported limited read access; details excluded.'] if check['permission_limitations'] else []
if 'limitations' in check:
clean['limitations'] = ['Collector reported scope limitations; consult the inventory owner.'] if check['limitations'] else []
checks[name] = clean
data['checks'] = checks
return restrict_inventory_fields(data)
return validate_inventory(minimize_inventory(data))

View File

@@ -1,56 +1,26 @@
"""Read-only local Linux inventory. No networking, sudo, shells or configuration changes."""
import json
import os
from pathlib import Path
import platform
import re
import shlex
import socket
import subprocess
import tempfile
import time
from datetime import datetime, timezone
from samsara.collectors.linux_schedules import read_system_cron, enrich_timers
class CollectionCancelled(Exception):
pass
from samsara.core.process import CollectionCancelled, capture
from samsara.core.privacy import minimize_inventory, load_privacy_policy
def run_command(arguments, cancelled=lambda: False, timeout=8):
"""Execute only fixed collector commands from system directories, with bounded output."""
executable = next((str(Path(folder) / arguments[0]) for folder in
('/usr/bin', '/usr/sbin', '/bin', '/sbin')
if (Path(folder) / arguments[0]).is_file()), None)
if executable is None:
raise RuntimeError('Required inventory command is unavailable.')
environment = {'PATH': '/usr/bin:/usr/sbin:/bin:/sbin', 'LC_ALL': 'C',
'LANG': 'C', 'SYSTEMD_PAGER': '', 'SYSTEMD_COLORS': '0'}
with tempfile.TemporaryFile() as output:
process = subprocess.Popen([executable, *arguments[1:]], stdout=output,
stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
env=environment, shell=False)
deadline = time.monotonic() + timeout
try:
while process.poll() is None:
if cancelled():
raise CollectionCancelled('Local collection cancelled.')
if time.monotonic() >= deadline:
raise RuntimeError('Inventory command timed out.')
if os.fstat(output.fileno()).st_size > 10 * 1024 * 1024:
raise RuntimeError('Inventory command exceeded its output limit.')
time.sleep(.05)
if process.returncode:
raise RuntimeError('Inventory command unavailable or access denied.')
output.seek(0)
raw = output.read(10 * 1024 * 1024 + 1)
if len(raw) > 10 * 1024 * 1024:
raise RuntimeError('Inventory command exceeded its output limit.')
return raw.decode('utf-8', errors='replace')
finally:
if process.poll() is None:
process.kill()
process.wait()
executable = next((str(Path(folder)/arguments[0]) for folder in
('/usr/bin','/usr/sbin','/bin','/sbin') if (Path(folder)/arguments[0]).is_file()), None)
if executable is None: raise RuntimeError('Required inventory command unavailable.')
environment = {'PATH':'/usr/bin:/usr/sbin:/bin:/sbin','LC_ALL':'C','LANG':'C',
'SYSTEMD_PAGER':'','SYSTEMD_COLORS':'0'}
return capture([executable,*arguments[1:]], env=environment, timeout=timeout,
cancelled=cancelled).decode('utf-8',errors='replace')
def parse_units(text, enabled_text, kind='service'):
@@ -74,6 +44,12 @@ def parse_units(text, enabled_text, kind='service'):
return rows
# ss(8) spellings mapped to the canonical names shared with the Windows collector.
TCP_STATES = {'LISTEN': 'Listen', 'ESTAB': 'Established', 'TIME-WAIT': 'TimeWait', 'CLOSE-WAIT': 'CloseWait',
'SYN-SENT': 'SynSent', 'SYN-RECV': 'SynReceived', 'FIN-WAIT-1': 'FinWait1',
'FIN-WAIT-2': 'FinWait2', 'LAST-ACK': 'LastAck', 'CLOSING': 'Closing'}
def endpoint(value):
address, port = value.rsplit(':', 1)
return address.strip('[]'), int(port) if port.isdigit() else 0
@@ -85,7 +61,7 @@ def parse_connections(text):
fields = line.split(None, 5)
if len(fields) < 5:
continue
state = {'LISTEN': 'Listen', 'ESTAB': 'Established'}.get(fields[0], fields[0])
state = TCP_STATES.get(fields[0], fields[0])
local, lp = endpoint(fields[3])
remote, rp = endpoint(fields[4])
process = re.search(r'\("([^"\n]+)",pid=(\d+)', fields[5] if len(fields) > 5 else '')
@@ -100,10 +76,11 @@ def parse_connections(text):
def collect_local_linux(stage=lambda message: None, cancelled=lambda: False, runner=run_command):
if platform.system() != 'Linux':
raise ValueError('Local Linux collection requires a Linux execution host.')
privacy_policy = load_privacy_policy()
started = time.monotonic()
data = {'schema_version': '1.0', 'synthetic': False,
'collected_at': datetime.now(timezone.utc).isoformat(),
'server': {'hostname': socket.gethostname(), 'platform': 'linux',
'server': {'hostname': socket.gethostname() if privacy_policy['approved_hostnames'] else 'Host-0001', 'platform': 'linux',
'collection_source': 'Local Linux read-only collection',
'operating_system': platform.system(), 'os_build': platform.release(),
'last_boot': 'Not collected', 'ip_addresses': [], 'network_adapters': [],
@@ -151,14 +128,9 @@ def collect_local_linux(stage=lambda message: None, cancelled=lambda: False, run
check('services', lambda: units('service'), 'System systemd units observed; per-service accounts, user units and executables not collected.')
def timers():
units('timer')
enrich_timers(data['inventory']['tasks'], command)
# Detailed unit properties may contain identifiers; omit by default.
check('tasks', timers, 'Loaded system timers and available calendar/next-run metadata observed; unloaded/user timers and application schedulers require review.')
def cron():
rows, incomplete = read_system_cron(cancelled)
data['inventory']['tasks'].extend(rows)
if incomplete:
raise RuntimeError('Some system cron files inaccessible')
check('cron', cron, 'Readable /etc/crontab and /etc/cron.d entries observed. Arguments omitted. User crontabs, anacron rules and periodic-script contents not inspected; timezone and actual execution unverified.')
data['checks']['cron'] = {'status':'not_collected','message':'Privacy policy: cron commands and script files are not read. Verify schedules separately.'}
def network():
data['inventory']['network'] = parse_connections(command(['ss', '-H', '-n', '-t', '-a', '-p']))
check('network', network, 'TCP snapshot only. Process visibility is limited by permissions; direction, historical activity, UDP and other namespaces not verified.')
@@ -192,7 +164,7 @@ def collect_local_linux(stage=lambda message: None, cancelled=lambda: False, run
}.items():
data['checks'][key] = {'status': 'not_collected', 'message': message}
check_cancel()
return data
return minimize_inventory(data, privacy_policy, trusted_source='Local Linux read-only collection')
def main():
@@ -204,8 +176,8 @@ def main():
try:
result = collect_local_linux()
# Exclusive creation prevents accidentally replacing an existing report.
with open(args.output, 'x', encoding='utf-8') as destination:
json.dump(result, destination, indent=2)
from samsara.core.storage import write_private
write_private(args.output, json.dumps(result, indent=2))
print('Local inventory saved. Review per-check statuses before making retirement decisions.')
except (OSError, ValueError, RuntimeError):
parser.exit(1, 'Collection/export failed. Check Linux platform, destination permissions and whether the destination already exists.\n')

View File

@@ -8,13 +8,13 @@ from pathlib import Path
import platform
import struct
import subprocess
import tempfile
import time
from jsonschema import Draft202012Validator, FormatChecker
from samsara.collectors.linux import CollectionCancelled
from samsara.core.process import CollectionCancelled, ProcessFailure, capture
from samsara.core.privacy import minimize_inventory, load_privacy_policy, safe_time
from samsara.core.detection import check_profile
from samsara.models.inventory import inventory_schema, validate_inventory, restrict_inventory_fields
from samsara.models.inventory import inventory_schema, validate_inventory
SCRIPT = Path(__file__).resolve().parents[2] / 'powershell' / 'Get-ServerInventory.ps1'
SECTIONS = ('system', 'roles', 'services', 'network', 'tasks', 'software', 'storage', 'shares', 'certificates')
@@ -56,44 +56,19 @@ def run_section(section, executable, cancelled=lambda: False, timeout=45, diagno
args = [str(executable), '-NoLogo', '-NoProfile', '-NonInteractive', '-File', str(SCRIPT), '-Section', section]
if diagnostic:
args.append('-Diagnostic')
environment = os.environ.copy()
# Only inbox modules are needed. Do not autoload user-supplied modules or profiles.
environment = {key: os.environ[key] for key in ('SystemRoot','WINDIR','TEMP','TMP','COMSPEC','SystemDrive') if key in os.environ}
environment['PSModulePath'] = str(Path(executable).parent / 'Modules')
with tempfile.TemporaryFile() as output:
try:
process = subprocess.Popen(args, stdin=subprocess.DEVNULL, stdout=output,
stderr=subprocess.DEVNULL, shell=False, env=environment,
creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
except OSError:
raise WindowsCollectionError('powershell_failed', 'Windows PowerShell could not start. Check executable permissions and local policy.') from None
deadline = time.monotonic() + timeout
try:
while process.poll() is None:
if cancelled():
raise CollectionCancelled()
if time.monotonic() >= deadline:
raise WindowsCollectionError('timeout', 'The local section exceeded its time limit.')
if os.fstat(output.fileno()).st_size > MAX_OUTPUT:
raise WindowsCollectionError('output_limit', 'The local section exceeded its output limit.')
time.sleep(.05)
if cancelled():
raise CollectionCancelled()
if process.returncode:
if process.returncode == 3:
raise WindowsCollectionError('serialization_failed', 'PowerShell could not serialize inventory metadata; no successful evidence was substituted.')
raise WindowsCollectionError('powershell_failed', 'The trusted script could not run. Check script signing policy, module availability and read permissions; raw diagnostics are excluded.')
output.seek(0)
raw = output.read(MAX_OUTPUT + 1)
if len(raw) > MAX_OUTPUT:
raise WindowsCollectionError('output_limit', 'The local section exceeded its output limit.')
try:
return json.loads(raw.decode('utf-8-sig'))
except (UnicodeError, ValueError, RecursionError):
raise WindowsCollectionError('invalid_output', 'PowerShell did not return a valid UTF-8 JSON section.') from None
finally:
if process.poll() is None:
process.kill()
process.wait()
try:
raw = capture(args, env=environment, timeout=timeout, cancelled=cancelled,
limit=MAX_OUTPUT, creationflags=getattr(subprocess,'CREATE_NO_WINDOW',0))
except ProcessFailure as exc:
if exc.returncode == 3:
raise WindowsCollectionError('serialization_failed', 'PowerShell could not serialize inventory metadata; no successful evidence was substituted.') from None
raise WindowsCollectionError(exc.code, str(exc)) from None
try:
return json.loads(raw.decode('utf-8-sig'))
except (UnicodeError, ValueError, RecursionError):
raise WindowsCollectionError('invalid_output','PowerShell did not return a valid JSON section.') from None
def validate_section(record, section):
@@ -108,7 +83,8 @@ def validate_section(record, section):
section_schema = {'type': 'object', 'required': ['section','status','message','started_at','completed_at','errors','permission_limitations','limitations','data'], 'properties': properties}
if not Draft202012Validator(section_schema, format_checker=FormatChecker()).is_valid(record):
raise WindowsCollectionError('invalid_output', 'PowerShell returned a section that violates the inventory contract.')
if record['completed_at'] < record['started_at']:
start, end = safe_time(record['started_at']), safe_time(record['completed_at'])
if start is None or end is None or end < start:
# Both native timestamps use the same UTC format. This is not a trust check.
raise WindowsCollectionError('invalid_output', 'PowerShell returned inconsistent section timestamps.')
return record
@@ -136,6 +112,7 @@ def collect_local_windows(stage=lambda message: None, cancelled=lambda: False, p
raise ValueError('Local Windows collection requires an authorized Windows Server execution host.')
if profile != 'auto' and not profile.startswith('windows:'):
raise ValueError('Select Auto-detect or a Windows Server profile for this execution host.')
load_privacy_policy()
executable = powershell_executable()
runner = runner or run_section
started = time.monotonic()
@@ -179,7 +156,7 @@ def collect_local_windows(stage=lambda message: None, cancelled=lambda: False, p
raise ValueError('Windows identity collection failed: ' + str(exc)) from None
if cancelled():
raise CollectionCancelled()
return validate_inventory(restrict_inventory_fields(result, local_source='Local Windows read-only collection'))
return validate_inventory(minimize_inventory(result, trusted_source='Local Windows read-only collection'))
def main():
@@ -190,8 +167,8 @@ def main():
args = parser.parse_args()
try:
data = collect_local_windows(profile=args.profile, diagnostic=args.diagnostic)
with open(args.output, 'x', encoding='utf-8') as output:
json.dump(data, output, indent=2)
from samsara.core.storage import write_private
write_private(args.output, json.dumps(data, indent=2))
print('Windows inventory saved. Review all failed/partial checks and permission limitations before retirement decisions.')
except (OSError, ValueError, WindowsCollectionError):
parser.exit(1, 'Windows inventory was not saved. Check supported local Server OS, Python architecture, script policy, permissions and a new writable output filename.\n')