ics-simlab-config-gen-claude/tests/test_semantic_validation_p0.py

615 lines
24 KiB
Python

#!/usr/bin/env python3
"""
Tests for P0 semantic validation: orphan devices, boolean type rules, null stripping.
These tests verify that configuration.json adheres to ICS-SimLab semantic invariants:
- All sensors must be monitored by at least one PLC
- All actuators must be controlled by at least one PLC
- Boolean signals must use coil/discrete_input, not input_register/holding_register
- Null fields should be stripped from output
"""
import json
from pathlib import Path
import pytest
from models.ics_simlab_config_v2 import Config, set_strict_mode
from services.patches import strip_nulls, sanitize_connection_id, patch_sanitize_connection_ids
from tools.semantic_validation import (
validate_orphan_devices,
validate_boolean_type_rules,
validate_all_semantics,
validate_plc_local_register_coherence,
)
from tools.repair_config import repair_plc_local_registers
FIXTURES_DIR = Path(__file__).parent / "fixtures"
class TestStripNulls:
"""Test strip_nulls canonicalization function."""
def test_strip_nulls_removes_none_keys(self):
"""Keys with None values should be removed."""
obj = {"a": 1, "b": None, "c": "hello"}
result = strip_nulls(obj)
assert result == {"a": 1, "c": "hello"}
assert "b" not in result
def test_strip_nulls_nested_dict(self):
"""Nulls should be removed at all nesting levels."""
obj = {
"outer": {
"keep": "value",
"remove": None,
"nested": {
"deep_keep": 42,
"deep_remove": None
}
}
}
result = strip_nulls(obj)
assert result == {
"outer": {
"keep": "value",
"nested": {
"deep_keep": 42
}
}
}
def test_strip_nulls_list_with_none_items(self):
"""None items in lists should be removed."""
obj = {"items": [1, None, 2, None, 3]}
result = strip_nulls(obj)
assert result == {"items": [1, 2, 3]}
def test_strip_nulls_list_of_dicts(self):
"""Dicts inside lists should have their nulls stripped."""
obj = {
"registers": [
{"id": "reg1", "io": None, "physical_value": None},
{"id": "reg2", "io": "input"}
]
}
result = strip_nulls(obj)
assert result == {
"registers": [
{"id": "reg1"},
{"id": "reg2", "io": "input"}
]
}
def test_strip_nulls_preserves_empty_structures(self):
"""Empty dicts and lists should be preserved (they're not None)."""
obj = {"empty_dict": {}, "empty_list": [], "value": "keep"}
result = strip_nulls(obj)
assert result == {"empty_dict": {}, "empty_list": [], "value": "keep"}
def test_strip_nulls_fixture(self):
"""Test on realistic fixture with many nulls."""
fixture_path = FIXTURES_DIR / "config_with_nulls.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
result = strip_nulls(raw)
# Check that plc1.network.port (null) is gone
assert "port" not in result["plcs"][0]["network"]
# Check that plc1.identity (null) is gone
assert "identity" not in result["plcs"][0]
# Check that inbound_connections[0].id (null) is gone
assert "id" not in result["plcs"][0]["inbound_connections"][0]
# Check that registers coil[0] has no physical_value/physical_values
assert "physical_value" not in result["plcs"][0]["registers"]["coil"][0]
assert "physical_values" not in result["plcs"][0]["registers"]["coil"][0]
class TestOrphanDeviceValidation:
"""Test orphan sensor/actuator detection."""
@pytest.fixture(autouse=True)
def reset_strict_mode(self):
"""Reset strict mode before each test."""
set_strict_mode(False)
yield
set_strict_mode(False)
def test_orphan_sensor_detected(self):
"""Sensor not referenced by any PLC monitor should fail."""
fixture_path = FIXTURES_DIR / "orphan_sensor.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
config = Config.model_validate(raw)
errors = validate_orphan_devices(config)
assert len(errors) == 1
assert "orphan_sensor" in str(errors[0]).lower()
assert "orphan" in str(errors[0]).lower()
def test_orphan_actuator_detected(self):
"""Actuator not referenced by any PLC controller should fail."""
fixture_path = FIXTURES_DIR / "orphan_actuator.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
config = Config.model_validate(raw)
errors = validate_orphan_devices(config)
assert len(errors) == 1
assert "orphan_actuator" in str(errors[0]).lower()
assert "orphan" in str(errors[0]).lower()
def test_valid_config_no_orphans(self):
"""Config with properly connected devices should pass."""
fixture_path = FIXTURES_DIR / "valid_minimal.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
config = Config.model_validate(raw)
errors = validate_orphan_devices(config)
assert len(errors) == 0, f"Unexpected errors: {errors}"
class TestBooleanTypeRules:
"""Test boolean signal type validation."""
@pytest.fixture(autouse=True)
def reset_strict_mode(self):
"""Reset strict mode before each test."""
set_strict_mode(False)
yield
set_strict_mode(False)
def test_boolean_in_input_register_detected(self):
"""Boolean signal in input_register should fail."""
fixture_path = FIXTURES_DIR / "boolean_type_wrong.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
config = Config.model_validate(raw)
errors = validate_boolean_type_rules(config)
assert len(errors) >= 1
# Should detect "bottle_at_filler_output" is a boolean in wrong register type
error_str = str(errors[0]).lower()
assert "boolean" in error_str or "discrete_input" in error_str
def test_valid_config_passes_boolean_check(self):
"""Config with correct boolean types should pass."""
fixture_path = FIXTURES_DIR / "valid_minimal.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
config = Config.model_validate(raw)
errors = validate_boolean_type_rules(config)
assert len(errors) == 0, f"Unexpected errors: {errors}"
class TestAllSemanticsIntegration:
"""Integration tests for validate_all_semantics."""
@pytest.fixture(autouse=True)
def reset_strict_mode(self):
"""Reset strict mode before each test."""
set_strict_mode(False)
yield
set_strict_mode(False)
def test_valid_config_passes_all_checks(self):
"""A properly configured system should pass all semantic checks."""
fixture_path = FIXTURES_DIR / "valid_minimal.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
config = Config.model_validate(raw)
errors = validate_all_semantics(config)
assert len(errors) == 0, f"Unexpected errors: {errors}"
def test_multiple_issues_detected(self):
"""Config with multiple issues should report all of them."""
# Create a config with both orphan and boolean issues
raw = {
"ui": {"network": {"ip": "192.168.0.1", "port": 5000, "docker_network": "vlan1"}},
"hmis": [],
"plcs": [{
"name": "plc1",
"logic": "plc1.py",
"network": {"ip": "192.168.0.21", "docker_network": "vlan1"},
"inbound_connections": [{"type": "tcp", "ip": "192.168.0.21", "port": 502}],
"outbound_connections": [],
"registers": {
"coil": [],
"discrete_input": [],
"holding_register": [],
"input_register": []
},
"monitors": [],
"controllers": []
}],
"sensors": [{
"name": "orphan_sensor",
"hil": "hil1",
"network": {"ip": "192.168.0.31", "docker_network": "vlan1"},
"inbound_connections": [{"type": "tcp", "ip": "192.168.0.31", "port": 502}],
"registers": {
"coil": [],
"discrete_input": [],
"holding_register": [],
"input_register": [{"address": 100, "count": 1, "physical_value": "switch_status"}]
}
}],
"actuators": [],
"hils": [{"name": "hil1", "logic": "hil1.py", "physical_values": [{"name": "switch_status", "io": "output"}]}],
"serial_networks": [],
"ip_networks": [{"docker_name": "vlan1", "name": "vlan1", "subnet": "192.168.0.0/24"}]
}
config = Config.model_validate(raw)
errors = validate_all_semantics(config)
# Should have at least 2 errors: orphan sensor + boolean type issue
assert len(errors) >= 2, f"Expected at least 2 errors, got {len(errors)}: {errors}"
class TestSanitizeConnectionId:
"""Test connection ID sanitization function."""
def test_sanitize_lowercase(self):
"""IDs should be lowercased."""
assert sanitize_connection_id("TO_SENSOR") == "to_sensor"
assert sanitize_connection_id("To-Sensor") == "to_sensor"
def test_sanitize_hyphens_to_underscore(self):
"""Hyphens should be converted to underscores."""
assert sanitize_connection_id("to-tank-sensor") == "to_tank_sensor"
assert sanitize_connection_id("To-Tank-Sensor") == "to_tank_sensor"
def test_sanitize_spaces_to_underscore(self):
"""Spaces should be converted to underscores."""
assert sanitize_connection_id("to sensor") == "to_sensor"
assert sanitize_connection_id("to sensor") == "to_sensor"
def test_sanitize_removes_special_chars(self):
"""Special characters should be removed."""
assert sanitize_connection_id("to@sensor!") == "tosensor"
assert sanitize_connection_id("to#$%sensor") == "tosensor"
def test_sanitize_collapses_underscores(self):
"""Multiple underscores should be collapsed."""
assert sanitize_connection_id("to__sensor") == "to_sensor"
assert sanitize_connection_id("to___sensor") == "to_sensor"
def test_sanitize_strips_underscores(self):
"""Leading/trailing underscores should be stripped."""
assert sanitize_connection_id("_to_sensor_") == "to_sensor"
assert sanitize_connection_id("__to_sensor__") == "to_sensor"
def test_sanitize_empty_fallback(self):
"""Empty string should fallback to 'connection'."""
assert sanitize_connection_id("") == "connection"
assert sanitize_connection_id(" ") == "connection"
assert sanitize_connection_id("@#$") == "connection"
def test_sanitize_already_valid(self):
"""Already valid IDs should be unchanged."""
assert sanitize_connection_id("to_sensor") == "to_sensor"
assert sanitize_connection_id("to_sensor1") == "to_sensor1"
class TestPatchSanitizeConnectionIds:
"""Test connection ID patch function."""
@pytest.fixture(autouse=True)
def reset_strict_mode(self):
"""Reset strict mode before each test."""
set_strict_mode(False)
yield
set_strict_mode(False)
def test_patch_sanitizes_outbound_connections(self):
"""Outbound connection IDs should be sanitized."""
cfg = {
"plcs": [{
"name": "plc1",
"outbound_connections": [
{"type": "tcp", "ip": "192.168.0.31", "port": 502, "id": "To-Sensor"},
{"type": "tcp", "ip": "192.168.0.41", "port": 502, "id": "TO_ACTUATOR"}
],
"monitors": [],
"controllers": []
}],
"hmis": []
}
patched, errors = patch_sanitize_connection_ids(cfg)
assert len(errors) == 0
assert patched["plcs"][0]["outbound_connections"][0]["id"] == "to_sensor"
assert patched["plcs"][0]["outbound_connections"][1]["id"] == "to_actuator"
def test_patch_updates_monitor_references(self):
"""Monitor outbound_connection_id references should be updated."""
cfg = {
"plcs": [{
"name": "plc1",
"outbound_connections": [
{"type": "tcp", "ip": "192.168.0.31", "port": 502, "id": "To-Sensor"}
],
"monitors": [
{"outbound_connection_id": "To-Sensor", "id": "tank_level", "value_type": "input_register", "address": 100}
],
"controllers": []
}],
"hmis": []
}
patched, errors = patch_sanitize_connection_ids(cfg)
assert len(errors) == 0
assert patched["plcs"][0]["monitors"][0]["outbound_connection_id"] == "to_sensor"
def test_patch_updates_controller_references(self):
"""Controller outbound_connection_id references should be updated."""
cfg = {
"plcs": [{
"name": "plc1",
"outbound_connections": [
{"type": "tcp", "ip": "192.168.0.41", "port": 502, "id": "TO_ACTUATOR"}
],
"monitors": [],
"controllers": [
{"outbound_connection_id": "TO_ACTUATOR", "id": "valve_cmd", "value_type": "coil", "address": 500}
]
}],
"hmis": []
}
patched, errors = patch_sanitize_connection_ids(cfg)
assert len(errors) == 0
assert patched["plcs"][0]["controllers"][0]["outbound_connection_id"] == "to_actuator"
def test_patch_from_fixture(self):
"""Test sanitization from fixture file."""
fixture_path = FIXTURES_DIR / "unsanitized_connection_ids.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
patched, errors = patch_sanitize_connection_ids(raw)
assert len(errors) == 0
# Check outbound_connection IDs are sanitized
assert patched["plcs"][0]["outbound_connections"][0]["id"] == "to_tank_sensor"
assert patched["plcs"][0]["outbound_connections"][1]["id"] == "to_valve_actuator"
# Check monitor/controller references are updated
assert patched["plcs"][0]["monitors"][0]["outbound_connection_id"] == "to_tank_sensor"
assert patched["plcs"][0]["controllers"][0]["outbound_connection_id"] == "to_valve_actuator"
class TestPlcLocalRegisterCoherence:
"""Test PLC local register coherence validation."""
@pytest.fixture(autouse=True)
def reset_strict_mode(self):
"""Reset strict mode before each test."""
set_strict_mode(False)
yield
set_strict_mode(False)
def test_missing_monitor_register_detected(self):
"""Missing local register for monitor should fail."""
fixture_path = FIXTURES_DIR / "plc_missing_local_registers.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
config = Config.model_validate(raw)
errors = validate_plc_local_register_coherence(config)
# Should detect missing registers for both monitor and controller
assert len(errors) >= 2
error_str = " ".join(str(e) for e in errors).lower()
assert "tank_level" in error_str
assert "valve_cmd" in error_str
def test_valid_config_passes_coherence(self):
"""Config with proper local registers should pass."""
fixture_path = FIXTURES_DIR / "valid_minimal.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
config = Config.model_validate(raw)
errors = validate_plc_local_register_coherence(config)
assert len(errors) == 0, f"Unexpected errors: {errors}"
def test_wrong_io_direction_detected(self):
"""Register with wrong io direction should fail."""
raw = {
"ui": {"network": {"ip": "192.168.0.1", "port": 5000, "docker_network": "vlan1"}},
"hmis": [],
"plcs": [{
"name": "plc1",
"logic": "plc1.py",
"network": {"ip": "192.168.0.21", "docker_network": "vlan1"},
"inbound_connections": [{"type": "tcp", "ip": "192.168.0.21", "port": 502}],
"outbound_connections": [
{"type": "tcp", "ip": "192.168.0.31", "port": 502, "id": "to_sensor"}
],
"registers": {
"coil": [],
"discrete_input": [],
"holding_register": [],
"input_register": [{"address": 100, "count": 1, "id": "tank_level", "io": "output"}]
},
"monitors": [
{"outbound_connection_id": "to_sensor", "id": "tank_level", "value_type": "input_register", "slave_id": 1, "address": 100, "count": 1, "interval": 0.5}
],
"controllers": []
}],
"sensors": [{
"name": "tank_sensor",
"hil": "hil1",
"network": {"ip": "192.168.0.31", "docker_network": "vlan1"},
"inbound_connections": [{"type": "tcp", "ip": "192.168.0.31", "port": 502}],
"registers": {
"coil": [],
"discrete_input": [],
"holding_register": [],
"input_register": [{"address": 100, "count": 1, "physical_value": "tank_level"}]
}
}],
"actuators": [],
"hils": [{"name": "hil1", "logic": "hil1.py", "physical_values": [{"name": "tank_level", "io": "output"}]}],
"serial_networks": [],
"ip_networks": [{"docker_name": "vlan1", "name": "vlan1", "subnet": "192.168.0.0/24"}]
}
config = Config.model_validate(raw)
errors = validate_plc_local_register_coherence(config)
assert len(errors) == 1
assert "io mismatch" in str(errors[0]).lower()
assert "output" in str(errors[0]).lower()
class TestRepairPlcLocalRegisters:
"""Test PLC local register repair function."""
@pytest.fixture(autouse=True)
def reset_strict_mode(self):
"""Reset strict mode before each test."""
set_strict_mode(False)
yield
set_strict_mode(False)
def test_repair_creates_monitor_register(self):
"""Repair should create missing register for monitor."""
cfg = {
"plcs": [{
"name": "plc1",
"registers": {
"coil": [],
"discrete_input": [],
"holding_register": [],
"input_register": []
},
"monitors": [
{"outbound_connection_id": "to_sensor", "id": "tank_level", "value_type": "input_register", "address": 100, "count": 1}
],
"controllers": []
}]
}
repaired, actions = repair_plc_local_registers(cfg)
assert len(actions) == 1
assert "tank_level" in str(actions[0])
assert "io='input'" in str(actions[0])
# Check register was created
input_regs = repaired["plcs"][0]["registers"]["input_register"]
assert len(input_regs) == 1
assert input_regs[0]["id"] == "tank_level"
assert input_regs[0]["io"] == "input"
assert input_regs[0]["address"] == 100
def test_repair_creates_controller_register(self):
"""Repair should create missing register for controller."""
cfg = {
"plcs": [{
"name": "plc1",
"registers": {
"coil": [],
"discrete_input": [],
"holding_register": [],
"input_register": []
},
"monitors": [],
"controllers": [
{"outbound_connection_id": "to_actuator", "id": "valve_cmd", "value_type": "coil", "address": 500, "count": 1}
]
}]
}
repaired, actions = repair_plc_local_registers(cfg)
assert len(actions) == 1
assert "valve_cmd" in str(actions[0])
assert "io='output'" in str(actions[0])
# Check register was created
coil_regs = repaired["plcs"][0]["registers"]["coil"]
assert len(coil_regs) == 1
assert coil_regs[0]["id"] == "valve_cmd"
assert coil_regs[0]["io"] == "output"
assert coil_regs[0]["address"] == 500
def test_repair_does_not_duplicate_existing(self):
"""Repair should not duplicate existing registers."""
cfg = {
"plcs": [{
"name": "plc1",
"registers": {
"coil": [],
"discrete_input": [],
"holding_register": [],
"input_register": [{"address": 100, "count": 1, "id": "tank_level", "io": "input"}]
},
"monitors": [
{"outbound_connection_id": "to_sensor", "id": "tank_level", "value_type": "input_register", "address": 100, "count": 1}
],
"controllers": []
}]
}
repaired, actions = repair_plc_local_registers(cfg)
assert len(actions) == 0
# Register count should be unchanged
assert len(repaired["plcs"][0]["registers"]["input_register"]) == 1
def test_repair_from_fixture(self):
"""Test repair from fixture file."""
fixture_path = FIXTURES_DIR / "plc_missing_local_registers.json"
if not fixture_path.exists():
pytest.skip(f"Fixture not found: {fixture_path}")
raw = json.loads(fixture_path.read_text(encoding="utf-8"))
repaired, actions = repair_plc_local_registers(raw)
# Should create 2 registers (1 for monitor, 1 for controller)
assert len(actions) == 2
# Verify input_register for monitor
input_regs = repaired["plcs"][0]["registers"]["input_register"]
tank_reg = [r for r in input_regs if r.get("id") == "tank_level"]
assert len(tank_reg) == 1
assert tank_reg[0]["io"] == "input"
# Verify coil for controller
coil_regs = repaired["plcs"][0]["registers"]["coil"]
valve_reg = [r for r in coil_regs if r.get("id") == "valve_cmd"]
assert len(valve_reg) == 1
assert valve_reg[0]["io"] == "output"
# Verify repaired config passes validation
config = Config.model_validate(repaired)
errors = validate_plc_local_register_coherence(config)
assert len(errors) == 0, f"Repair did not fix issues: {errors}"