Privacy hardening, remote-peer IP masking and review fixes
- Add privacy projection (core/privacy.py), private report storage, bounded
process capture; omit identities, free text and unapproved paths
- Mask remote peer IPs: only loopback/host/wildcard stay numeric, others are
reduced to remote_scope private/public (schema + scoring updated)
- Map ss TCP states to canonical names; ASCII-only numeric fields; fix
"Microsoft Visual C++" product matching (Python and PowerShell)
- Merge 7fed3b9: -Diagnostic switch, serialization exit code 3, PS7 module
path, UI error detail; keep hardened task/certificate collectors
- Add audit docs, privacy docs and regression tests
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,53 +1,31 @@
|
||||
"""Offline intake shared by Windows and Linux desktops. No code is executed."""
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
from samsara.models.inventory import validate_inventory, restrict_inventory_fields
|
||||
from samsara.core.privacy import minimize_inventory
|
||||
from samsara.core.storage import require_local_path
|
||||
from samsara.models.inventory import validate_inventory
|
||||
|
||||
MAX_IMPORT_BYTES = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def load_inventory(path):
|
||||
with Path(path).open('rb') as source:
|
||||
local = require_local_path(path)
|
||||
flags=os.O_RDONLY|getattr(os,'O_NOFOLLOW',0)|getattr(os,'O_NONBLOCK',0)|getattr(os,'O_BINARY',0)
|
||||
with os.fdopen(os.open(local,flags),'rb') as source:
|
||||
if not stat.S_ISREG(os.fstat(source.fileno()).st_mode):
|
||||
raise ValueError('Only regular local inventory files may be imported.')
|
||||
raw = source.read(MAX_IMPORT_BYTES + 1)
|
||||
if len(raw) > MAX_IMPORT_BYTES:
|
||||
raise ValueError('Inventory exceeds the 10 MiB import limit.')
|
||||
try:
|
||||
data = json.loads(raw.decode('utf-8-sig'))
|
||||
except (ValueError, UnicodeError):
|
||||
except (ValueError, UnicodeError, RecursionError):
|
||||
raise ValueError('The selected file is not valid UTF-8 inventory JSON.') from None
|
||||
# A report wrapper is accepted, but its precomputed assessment is never trusted.
|
||||
if isinstance(data, dict) and 'inventory_result' in data:
|
||||
data = data['inventory_result']
|
||||
validate_inventory(data)
|
||||
# Keep diagnostic contents out of logs and exported reports. Diagnostics may
|
||||
# contain command lines or credentials from external collection tools.
|
||||
safe_errors = {
|
||||
'access_denied': 'Current account could not read this scope.',
|
||||
'command_unavailable': 'Required command was unavailable.',
|
||||
'timeout': 'Section time limit exceeded.',
|
||||
'overall_timeout': 'Overall collection time limit exceeded.',
|
||||
'output_limit': 'Collection output limit exceeded.',
|
||||
'powershell_failed': 'PowerShell could not execute the collector.',
|
||||
'invalid_output': 'Invalid section output.',
|
||||
'collection_failed': 'A local query failed.'
|
||||
}
|
||||
checks = {}
|
||||
for name, check in data['checks'].items():
|
||||
clean = {'status': check['status'], 'message':
|
||||
'Imported check status. Source assertions are unverified; raw diagnostics excluded.'}
|
||||
for field in ('started_at', 'completed_at'):
|
||||
if field in check:
|
||||
clean[field] = check[field]
|
||||
if 'errors' in check:
|
||||
clean['errors'] = []
|
||||
for error in check['errors']:
|
||||
code = error['code'] if error['code'] in safe_errors else 'collection_failed'
|
||||
clean['errors'].append({'code': code, 'scope': 'Imported check', 'message': safe_errors[code]})
|
||||
if 'permission_limitations' in check:
|
||||
clean['permission_limitations'] = ['Collector reported limited read access; details excluded.'] if check['permission_limitations'] else []
|
||||
if 'limitations' in check:
|
||||
clean['limitations'] = ['Collector reported scope limitations; consult the inventory owner.'] if check['limitations'] else []
|
||||
checks[name] = clean
|
||||
data['checks'] = checks
|
||||
return restrict_inventory_fields(data)
|
||||
return validate_inventory(minimize_inventory(data))
|
||||
|
||||
@@ -1,56 +1,26 @@
|
||||
"""Read-only local Linux inventory. No networking, sudo, shells or configuration changes."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from samsara.collectors.linux_schedules import read_system_cron, enrich_timers
|
||||
|
||||
|
||||
class CollectionCancelled(Exception):
|
||||
pass
|
||||
from samsara.core.process import CollectionCancelled, capture
|
||||
from samsara.core.privacy import minimize_inventory, load_privacy_policy
|
||||
|
||||
|
||||
def run_command(arguments, cancelled=lambda: False, timeout=8):
|
||||
"""Execute only fixed collector commands from system directories, with bounded output."""
|
||||
executable = next((str(Path(folder) / arguments[0]) for folder in
|
||||
('/usr/bin', '/usr/sbin', '/bin', '/sbin')
|
||||
if (Path(folder) / arguments[0]).is_file()), None)
|
||||
if executable is None:
|
||||
raise RuntimeError('Required inventory command is unavailable.')
|
||||
environment = {'PATH': '/usr/bin:/usr/sbin:/bin:/sbin', 'LC_ALL': 'C',
|
||||
'LANG': 'C', 'SYSTEMD_PAGER': '', 'SYSTEMD_COLORS': '0'}
|
||||
with tempfile.TemporaryFile() as output:
|
||||
process = subprocess.Popen([executable, *arguments[1:]], stdout=output,
|
||||
stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
|
||||
env=environment, shell=False)
|
||||
deadline = time.monotonic() + timeout
|
||||
try:
|
||||
while process.poll() is None:
|
||||
if cancelled():
|
||||
raise CollectionCancelled('Local collection cancelled.')
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError('Inventory command timed out.')
|
||||
if os.fstat(output.fileno()).st_size > 10 * 1024 * 1024:
|
||||
raise RuntimeError('Inventory command exceeded its output limit.')
|
||||
time.sleep(.05)
|
||||
if process.returncode:
|
||||
raise RuntimeError('Inventory command unavailable or access denied.')
|
||||
output.seek(0)
|
||||
raw = output.read(10 * 1024 * 1024 + 1)
|
||||
if len(raw) > 10 * 1024 * 1024:
|
||||
raise RuntimeError('Inventory command exceeded its output limit.')
|
||||
return raw.decode('utf-8', errors='replace')
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
executable = next((str(Path(folder)/arguments[0]) for folder in
|
||||
('/usr/bin','/usr/sbin','/bin','/sbin') if (Path(folder)/arguments[0]).is_file()), None)
|
||||
if executable is None: raise RuntimeError('Required inventory command unavailable.')
|
||||
environment = {'PATH':'/usr/bin:/usr/sbin:/bin:/sbin','LC_ALL':'C','LANG':'C',
|
||||
'SYSTEMD_PAGER':'','SYSTEMD_COLORS':'0'}
|
||||
return capture([executable,*arguments[1:]], env=environment, timeout=timeout,
|
||||
cancelled=cancelled).decode('utf-8',errors='replace')
|
||||
|
||||
|
||||
def parse_units(text, enabled_text, kind='service'):
|
||||
@@ -74,6 +44,12 @@ def parse_units(text, enabled_text, kind='service'):
|
||||
return rows
|
||||
|
||||
|
||||
# ss(8) spellings mapped to the canonical names shared with the Windows collector.
|
||||
TCP_STATES = {'LISTEN': 'Listen', 'ESTAB': 'Established', 'TIME-WAIT': 'TimeWait', 'CLOSE-WAIT': 'CloseWait',
|
||||
'SYN-SENT': 'SynSent', 'SYN-RECV': 'SynReceived', 'FIN-WAIT-1': 'FinWait1',
|
||||
'FIN-WAIT-2': 'FinWait2', 'LAST-ACK': 'LastAck', 'CLOSING': 'Closing'}
|
||||
|
||||
|
||||
def endpoint(value):
|
||||
address, port = value.rsplit(':', 1)
|
||||
return address.strip('[]'), int(port) if port.isdigit() else 0
|
||||
@@ -85,7 +61,7 @@ def parse_connections(text):
|
||||
fields = line.split(None, 5)
|
||||
if len(fields) < 5:
|
||||
continue
|
||||
state = {'LISTEN': 'Listen', 'ESTAB': 'Established'}.get(fields[0], fields[0])
|
||||
state = TCP_STATES.get(fields[0], fields[0])
|
||||
local, lp = endpoint(fields[3])
|
||||
remote, rp = endpoint(fields[4])
|
||||
process = re.search(r'\("([^"\n]+)",pid=(\d+)', fields[5] if len(fields) > 5 else '')
|
||||
@@ -100,10 +76,11 @@ def parse_connections(text):
|
||||
def collect_local_linux(stage=lambda message: None, cancelled=lambda: False, runner=run_command):
|
||||
if platform.system() != 'Linux':
|
||||
raise ValueError('Local Linux collection requires a Linux execution host.')
|
||||
privacy_policy = load_privacy_policy()
|
||||
started = time.monotonic()
|
||||
data = {'schema_version': '1.0', 'synthetic': False,
|
||||
'collected_at': datetime.now(timezone.utc).isoformat(),
|
||||
'server': {'hostname': socket.gethostname(), 'platform': 'linux',
|
||||
'server': {'hostname': socket.gethostname() if privacy_policy['approved_hostnames'] else 'Host-0001', 'platform': 'linux',
|
||||
'collection_source': 'Local Linux read-only collection',
|
||||
'operating_system': platform.system(), 'os_build': platform.release(),
|
||||
'last_boot': 'Not collected', 'ip_addresses': [], 'network_adapters': [],
|
||||
@@ -151,14 +128,9 @@ def collect_local_linux(stage=lambda message: None, cancelled=lambda: False, run
|
||||
check('services', lambda: units('service'), 'System systemd units observed; per-service accounts, user units and executables not collected.')
|
||||
def timers():
|
||||
units('timer')
|
||||
enrich_timers(data['inventory']['tasks'], command)
|
||||
# Detailed unit properties may contain identifiers; omit by default.
|
||||
check('tasks', timers, 'Loaded system timers and available calendar/next-run metadata observed; unloaded/user timers and application schedulers require review.')
|
||||
def cron():
|
||||
rows, incomplete = read_system_cron(cancelled)
|
||||
data['inventory']['tasks'].extend(rows)
|
||||
if incomplete:
|
||||
raise RuntimeError('Some system cron files inaccessible')
|
||||
check('cron', cron, 'Readable /etc/crontab and /etc/cron.d entries observed. Arguments omitted. User crontabs, anacron rules and periodic-script contents not inspected; timezone and actual execution unverified.')
|
||||
data['checks']['cron'] = {'status':'not_collected','message':'Privacy policy: cron commands and script files are not read. Verify schedules separately.'}
|
||||
def network():
|
||||
data['inventory']['network'] = parse_connections(command(['ss', '-H', '-n', '-t', '-a', '-p']))
|
||||
check('network', network, 'TCP snapshot only. Process visibility is limited by permissions; direction, historical activity, UDP and other namespaces not verified.')
|
||||
@@ -192,7 +164,7 @@ def collect_local_linux(stage=lambda message: None, cancelled=lambda: False, run
|
||||
}.items():
|
||||
data['checks'][key] = {'status': 'not_collected', 'message': message}
|
||||
check_cancel()
|
||||
return data
|
||||
return minimize_inventory(data, privacy_policy, trusted_source='Local Linux read-only collection')
|
||||
|
||||
|
||||
def main():
|
||||
@@ -204,8 +176,8 @@ def main():
|
||||
try:
|
||||
result = collect_local_linux()
|
||||
# Exclusive creation prevents accidentally replacing an existing report.
|
||||
with open(args.output, 'x', encoding='utf-8') as destination:
|
||||
json.dump(result, destination, indent=2)
|
||||
from samsara.core.storage import write_private
|
||||
write_private(args.output, json.dumps(result, indent=2))
|
||||
print('Local inventory saved. Review per-check statuses before making retirement decisions.')
|
||||
except (OSError, ValueError, RuntimeError):
|
||||
parser.exit(1, 'Collection/export failed. Check Linux platform, destination permissions and whether the destination already exists.\n')
|
||||
|
||||
@@ -8,13 +8,13 @@ from pathlib import Path
|
||||
import platform
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from jsonschema import Draft202012Validator, FormatChecker
|
||||
from samsara.collectors.linux import CollectionCancelled
|
||||
from samsara.core.process import CollectionCancelled, ProcessFailure, capture
|
||||
from samsara.core.privacy import minimize_inventory, load_privacy_policy, safe_time
|
||||
from samsara.core.detection import check_profile
|
||||
from samsara.models.inventory import inventory_schema, validate_inventory, restrict_inventory_fields
|
||||
from samsara.models.inventory import inventory_schema, validate_inventory
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[2] / 'powershell' / 'Get-ServerInventory.ps1'
|
||||
SECTIONS = ('system', 'roles', 'services', 'network', 'tasks', 'software', 'storage', 'shares', 'certificates')
|
||||
@@ -56,44 +56,19 @@ def run_section(section, executable, cancelled=lambda: False, timeout=45, diagno
|
||||
args = [str(executable), '-NoLogo', '-NoProfile', '-NonInteractive', '-File', str(SCRIPT), '-Section', section]
|
||||
if diagnostic:
|
||||
args.append('-Diagnostic')
|
||||
environment = os.environ.copy()
|
||||
# Only inbox modules are needed. Do not autoload user-supplied modules or profiles.
|
||||
environment = {key: os.environ[key] for key in ('SystemRoot','WINDIR','TEMP','TMP','COMSPEC','SystemDrive') if key in os.environ}
|
||||
environment['PSModulePath'] = str(Path(executable).parent / 'Modules')
|
||||
with tempfile.TemporaryFile() as output:
|
||||
try:
|
||||
process = subprocess.Popen(args, stdin=subprocess.DEVNULL, stdout=output,
|
||||
stderr=subprocess.DEVNULL, shell=False, env=environment,
|
||||
creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
|
||||
except OSError:
|
||||
raise WindowsCollectionError('powershell_failed', 'Windows PowerShell could not start. Check executable permissions and local policy.') from None
|
||||
deadline = time.monotonic() + timeout
|
||||
try:
|
||||
while process.poll() is None:
|
||||
if cancelled():
|
||||
raise CollectionCancelled()
|
||||
if time.monotonic() >= deadline:
|
||||
raise WindowsCollectionError('timeout', 'The local section exceeded its time limit.')
|
||||
if os.fstat(output.fileno()).st_size > MAX_OUTPUT:
|
||||
raise WindowsCollectionError('output_limit', 'The local section exceeded its output limit.')
|
||||
time.sleep(.05)
|
||||
if cancelled():
|
||||
raise CollectionCancelled()
|
||||
if process.returncode:
|
||||
if process.returncode == 3:
|
||||
raise WindowsCollectionError('serialization_failed', 'PowerShell could not serialize inventory metadata; no successful evidence was substituted.')
|
||||
raise WindowsCollectionError('powershell_failed', 'The trusted script could not run. Check script signing policy, module availability and read permissions; raw diagnostics are excluded.')
|
||||
output.seek(0)
|
||||
raw = output.read(MAX_OUTPUT + 1)
|
||||
if len(raw) > MAX_OUTPUT:
|
||||
raise WindowsCollectionError('output_limit', 'The local section exceeded its output limit.')
|
||||
try:
|
||||
return json.loads(raw.decode('utf-8-sig'))
|
||||
except (UnicodeError, ValueError, RecursionError):
|
||||
raise WindowsCollectionError('invalid_output', 'PowerShell did not return a valid UTF-8 JSON section.') from None
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
try:
|
||||
raw = capture(args, env=environment, timeout=timeout, cancelled=cancelled,
|
||||
limit=MAX_OUTPUT, creationflags=getattr(subprocess,'CREATE_NO_WINDOW',0))
|
||||
except ProcessFailure as exc:
|
||||
if exc.returncode == 3:
|
||||
raise WindowsCollectionError('serialization_failed', 'PowerShell could not serialize inventory metadata; no successful evidence was substituted.') from None
|
||||
raise WindowsCollectionError(exc.code, str(exc)) from None
|
||||
try:
|
||||
return json.loads(raw.decode('utf-8-sig'))
|
||||
except (UnicodeError, ValueError, RecursionError):
|
||||
raise WindowsCollectionError('invalid_output','PowerShell did not return a valid JSON section.') from None
|
||||
|
||||
|
||||
def validate_section(record, section):
|
||||
@@ -108,7 +83,8 @@ def validate_section(record, section):
|
||||
section_schema = {'type': 'object', 'required': ['section','status','message','started_at','completed_at','errors','permission_limitations','limitations','data'], 'properties': properties}
|
||||
if not Draft202012Validator(section_schema, format_checker=FormatChecker()).is_valid(record):
|
||||
raise WindowsCollectionError('invalid_output', 'PowerShell returned a section that violates the inventory contract.')
|
||||
if record['completed_at'] < record['started_at']:
|
||||
start, end = safe_time(record['started_at']), safe_time(record['completed_at'])
|
||||
if start is None or end is None or end < start:
|
||||
# Both native timestamps use the same UTC format. This is not a trust check.
|
||||
raise WindowsCollectionError('invalid_output', 'PowerShell returned inconsistent section timestamps.')
|
||||
return record
|
||||
@@ -136,6 +112,7 @@ def collect_local_windows(stage=lambda message: None, cancelled=lambda: False, p
|
||||
raise ValueError('Local Windows collection requires an authorized Windows Server execution host.')
|
||||
if profile != 'auto' and not profile.startswith('windows:'):
|
||||
raise ValueError('Select Auto-detect or a Windows Server profile for this execution host.')
|
||||
load_privacy_policy()
|
||||
executable = powershell_executable()
|
||||
runner = runner or run_section
|
||||
started = time.monotonic()
|
||||
@@ -179,7 +156,7 @@ def collect_local_windows(stage=lambda message: None, cancelled=lambda: False, p
|
||||
raise ValueError('Windows identity collection failed: ' + str(exc)) from None
|
||||
if cancelled():
|
||||
raise CollectionCancelled()
|
||||
return validate_inventory(restrict_inventory_fields(result, local_source='Local Windows read-only collection'))
|
||||
return validate_inventory(minimize_inventory(result, trusted_source='Local Windows read-only collection'))
|
||||
|
||||
|
||||
def main():
|
||||
@@ -190,8 +167,8 @@ def main():
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
data = collect_local_windows(profile=args.profile, diagnostic=args.diagnostic)
|
||||
with open(args.output, 'x', encoding='utf-8') as output:
|
||||
json.dump(data, output, indent=2)
|
||||
from samsara.core.storage import write_private
|
||||
write_private(args.output, json.dumps(data, indent=2))
|
||||
print('Windows inventory saved. Review all failed/partial checks and permission limitations before retirement decisions.')
|
||||
except (OSError, ValueError, WindowsCollectionError):
|
||||
parser.exit(1, 'Windows inventory was not saved. Check supported local Server OS, Python architecture, script policy, permissions and a new writable output filename.\n')
|
||||
|
||||
Reference in New Issue
Block a user