412 lines
16 KiB
Python
412 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Tests for ICS-SimLab configuration validation.
|
|
|
|
Tests cover:
|
|
1. Pydantic validation of all example configurations
|
|
2. Type coercion (port/slave_id as string -> int)
|
|
3. Enrichment idempotency
|
|
4. Semantic validation error detection
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from models.ics_simlab_config_v2 import Config, set_strict_mode
|
|
from tools.enrich_config import enrich_plc_connections, enrich_hmi_connections
|
|
from tools.semantic_validation import validate_hmi_semantics
|
|
|
|
|
|
# Path to examples directory
|
|
EXAMPLES_DIR = Path(__file__).parent.parent / "examples"
|
|
|
|
|
|
class TestPydanticValidation:
|
|
"""Test that all example configs pass Pydantic validation."""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def reset_strict_mode(self):
|
|
"""Reset strict mode before each test."""
|
|
set_strict_mode(False)
|
|
yield
|
|
set_strict_mode(False)
|
|
|
|
@pytest.mark.parametrize("config_path", [
|
|
EXAMPLES_DIR / "water_tank" / "configuration.json",
|
|
EXAMPLES_DIR / "smart_grid" / "logic" / "configuration.json",
|
|
EXAMPLES_DIR / "ied" / "logic" / "configuration.json",
|
|
])
|
|
def test_example_validates(self, config_path: Path):
|
|
"""Each example configuration should pass Pydantic validation."""
|
|
if not config_path.exists():
|
|
pytest.skip(f"Example not found: {config_path}")
|
|
|
|
raw = json.loads(config_path.read_text(encoding="utf-8"))
|
|
config = Config.model_validate(raw)
|
|
|
|
# Basic sanity checks
|
|
assert config.ui is not None
|
|
assert len(config.plcs) >= 1 or len(config.hils) >= 1
|
|
|
|
def test_type_coercion_port_string(self):
|
|
"""port: '502' should be coerced to port: 502."""
|
|
raw = {
|
|
"ui": {"network": {"ip": "192.168.0.1", "port": 8501, "docker_network": "vlan1"}},
|
|
"plcs": [{
|
|
"name": "plc1",
|
|
"logic": "plc1.py",
|
|
"network": {"ip": "192.168.0.21", "docker_network": "vlan1"},
|
|
"outbound_connections": [
|
|
{"type": "tcp", "ip": "192.168.0.22", "port": "502", "id": "conn1"}
|
|
],
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [], "input_register": []
|
|
}
|
|
}],
|
|
"hmis": [], "sensors": [], "actuators": [], "hils": [],
|
|
"serial_networks": [], "ip_networks": []
|
|
}
|
|
config = Config.model_validate(raw)
|
|
|
|
assert config.plcs[0].outbound_connections[0].port == 502
|
|
assert isinstance(config.plcs[0].outbound_connections[0].port, int)
|
|
|
|
def test_type_coercion_slave_id_string(self):
|
|
"""slave_id: '1' should be coerced to slave_id: 1."""
|
|
raw = {
|
|
"ui": {"network": {"ip": "192.168.0.1", "port": 8501, "docker_network": "vlan1"}},
|
|
"plcs": [{
|
|
"name": "plc1",
|
|
"logic": "plc1.py",
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [], "input_register": []
|
|
},
|
|
"monitors": [{
|
|
"outbound_connection_id": "conn1",
|
|
"id": "reg1",
|
|
"value_type": "input_register",
|
|
"slave_id": "1",
|
|
"address": 1,
|
|
"count": 1,
|
|
"interval": 0.5
|
|
}]
|
|
}],
|
|
"hmis": [], "sensors": [], "actuators": [], "hils": [],
|
|
"serial_networks": [], "ip_networks": []
|
|
}
|
|
config = Config.model_validate(raw)
|
|
|
|
assert config.plcs[0].monitors[0].slave_id == 1
|
|
assert isinstance(config.plcs[0].monitors[0].slave_id, int)
|
|
|
|
def test_strict_mode_rejects_string_port(self):
|
|
"""In strict mode, string port should be rejected."""
|
|
set_strict_mode(True)
|
|
|
|
raw = {
|
|
"ui": {"network": {"ip": "192.168.0.1", "port": 8501, "docker_network": "vlan1"}},
|
|
"plcs": [{
|
|
"name": "plc1",
|
|
"logic": "plc1.py",
|
|
"network": {"ip": "192.168.0.21", "docker_network": "vlan1"},
|
|
"outbound_connections": [
|
|
{"type": "tcp", "ip": "192.168.0.22", "port": "502", "id": "conn1"}
|
|
],
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [], "input_register": []
|
|
}
|
|
}],
|
|
"hmis": [], "sensors": [], "actuators": [], "hils": [],
|
|
"serial_networks": [], "ip_networks": []
|
|
}
|
|
|
|
with pytest.raises(Exception) as exc_info:
|
|
Config.model_validate(raw)
|
|
|
|
assert "strict mode" in str(exc_info.value).lower()
|
|
|
|
def test_non_numeric_string_rejected(self):
|
|
"""Non-numeric strings like 'abc' should be rejected even in non-strict mode."""
|
|
raw = {
|
|
"ui": {"network": {"ip": "192.168.0.1", "port": 8501, "docker_network": "vlan1"}},
|
|
"plcs": [{
|
|
"name": "plc1",
|
|
"logic": "plc1.py",
|
|
"network": {"ip": "192.168.0.21", "docker_network": "vlan1"},
|
|
"outbound_connections": [
|
|
{"type": "tcp", "ip": "192.168.0.22", "port": "abc", "id": "conn1"}
|
|
],
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [], "input_register": []
|
|
}
|
|
}],
|
|
"hmis": [], "sensors": [], "actuators": [], "hils": [],
|
|
"serial_networks": [], "ip_networks": []
|
|
}
|
|
|
|
with pytest.raises(Exception) as exc_info:
|
|
Config.model_validate(raw)
|
|
|
|
assert "not strictly numeric" in str(exc_info.value).lower()
|
|
|
|
|
|
class TestEnrichIdempotency:
|
|
"""Test that enrichment is idempotent (running twice gives same result)."""
|
|
|
|
@pytest.mark.parametrize("config_path", [
|
|
EXAMPLES_DIR / "water_tank" / "configuration.json",
|
|
EXAMPLES_DIR / "smart_grid" / "logic" / "configuration.json",
|
|
EXAMPLES_DIR / "ied" / "logic" / "configuration.json",
|
|
])
|
|
def test_enrich_idempotent(self, config_path: Path):
|
|
"""Running enrich twice should produce identical output."""
|
|
if not config_path.exists():
|
|
pytest.skip(f"Example not found: {config_path}")
|
|
|
|
raw = json.loads(config_path.read_text(encoding="utf-8"))
|
|
|
|
# First enrichment
|
|
enriched1 = enrich_plc_connections(dict(raw))
|
|
enriched1 = enrich_hmi_connections(enriched1)
|
|
|
|
# Second enrichment
|
|
enriched2 = enrich_plc_connections(dict(enriched1))
|
|
enriched2 = enrich_hmi_connections(enriched2)
|
|
|
|
# Should be identical (compare as JSON to ignore dict ordering)
|
|
json1 = json.dumps(enriched1, sort_keys=True)
|
|
json2 = json.dumps(enriched2, sort_keys=True)
|
|
|
|
assert json1 == json2, "Enrichment is not idempotent"
|
|
|
|
|
|
class TestSemanticValidation:
|
|
"""Test semantic validation of HMI monitors/controllers."""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def reset_strict_mode(self):
|
|
"""Reset strict mode before each test."""
|
|
set_strict_mode(False)
|
|
yield
|
|
set_strict_mode(False)
|
|
|
|
def test_invalid_outbound_connection_detected(self):
|
|
"""Monitor with invalid outbound_connection_id should error."""
|
|
raw = {
|
|
"ui": {"network": {"ip": "192.168.0.1", "port": 8501, "docker_network": "vlan1"}},
|
|
"hmis": [{
|
|
"name": "hmi1",
|
|
"network": {"ip": "192.168.0.31", "docker_network": "vlan1"},
|
|
"inbound_connections": [],
|
|
"outbound_connections": [
|
|
{"type": "tcp", "ip": "192.168.0.21", "port": 502, "id": "plc1_con"}
|
|
],
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [], "input_register": []
|
|
},
|
|
"monitors": [{
|
|
"outbound_connection_id": "nonexistent_con",
|
|
"id": "tank_level",
|
|
"value_type": "input_register",
|
|
"slave_id": 1,
|
|
"address": 1,
|
|
"count": 1,
|
|
"interval": 0.5
|
|
}],
|
|
"controllers": []
|
|
}],
|
|
"plcs": [], "sensors": [], "actuators": [], "hils": [],
|
|
"serial_networks": [], "ip_networks": []
|
|
}
|
|
config = Config.model_validate(raw)
|
|
errors = validate_hmi_semantics(config)
|
|
|
|
assert len(errors) == 1
|
|
assert "nonexistent_con" in str(errors[0])
|
|
assert "not found" in str(errors[0]).lower()
|
|
|
|
def test_target_ip_not_found_detected(self):
|
|
"""Monitor targeting unknown IP should error."""
|
|
raw = {
|
|
"ui": {"network": {"ip": "192.168.0.1", "port": 8501, "docker_network": "vlan1"}},
|
|
"hmis": [{
|
|
"name": "hmi1",
|
|
"network": {"ip": "192.168.0.31", "docker_network": "vlan1"},
|
|
"inbound_connections": [],
|
|
"outbound_connections": [
|
|
{"type": "tcp", "ip": "192.168.0.99", "port": 502, "id": "unknown_con"}
|
|
],
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [], "input_register": []
|
|
},
|
|
"monitors": [{
|
|
"outbound_connection_id": "unknown_con",
|
|
"id": "tank_level",
|
|
"value_type": "input_register",
|
|
"slave_id": 1,
|
|
"address": 1,
|
|
"count": 1,
|
|
"interval": 0.5
|
|
}],
|
|
"controllers": []
|
|
}],
|
|
"plcs": [], "sensors": [], "actuators": [], "hils": [],
|
|
"serial_networks": [], "ip_networks": []
|
|
}
|
|
config = Config.model_validate(raw)
|
|
errors = validate_hmi_semantics(config)
|
|
|
|
assert len(errors) == 1
|
|
assert "192.168.0.99" in str(errors[0])
|
|
assert "not found" in str(errors[0]).lower()
|
|
|
|
def test_register_not_found_detected(self):
|
|
"""Monitor referencing nonexistent register should error."""
|
|
raw = {
|
|
"ui": {"network": {"ip": "192.168.0.1", "port": 8501, "docker_network": "vlan1"}},
|
|
"hmis": [{
|
|
"name": "hmi1",
|
|
"network": {"ip": "192.168.0.31", "docker_network": "vlan1"},
|
|
"inbound_connections": [],
|
|
"outbound_connections": [
|
|
{"type": "tcp", "ip": "192.168.0.21", "port": 502, "id": "plc1_con"}
|
|
],
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [], "input_register": []
|
|
},
|
|
"monitors": [{
|
|
"outbound_connection_id": "plc1_con",
|
|
"id": "nonexistent_register",
|
|
"value_type": "input_register",
|
|
"slave_id": 1,
|
|
"address": 1,
|
|
"count": 1,
|
|
"interval": 0.5
|
|
}],
|
|
"controllers": []
|
|
}],
|
|
"plcs": [{
|
|
"name": "plc1",
|
|
"logic": "plc1.py",
|
|
"network": {"ip": "192.168.0.21", "docker_network": "vlan1"},
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [],
|
|
"input_register": [
|
|
{"address": 1, "count": 1, "io": "input", "id": "tank_level"}
|
|
]
|
|
}
|
|
}],
|
|
"sensors": [], "actuators": [], "hils": [],
|
|
"serial_networks": [], "ip_networks": []
|
|
}
|
|
config = Config.model_validate(raw)
|
|
errors = validate_hmi_semantics(config)
|
|
|
|
assert len(errors) == 1
|
|
assert "nonexistent_register" in str(errors[0])
|
|
assert "not found" in str(errors[0]).lower()
|
|
|
|
def test_value_type_mismatch_detected(self):
|
|
"""Monitor with wrong value_type should error."""
|
|
raw = {
|
|
"ui": {"network": {"ip": "192.168.0.1", "port": 8501, "docker_network": "vlan1"}},
|
|
"hmis": [{
|
|
"name": "hmi1",
|
|
"network": {"ip": "192.168.0.31", "docker_network": "vlan1"},
|
|
"inbound_connections": [],
|
|
"outbound_connections": [
|
|
{"type": "tcp", "ip": "192.168.0.21", "port": 502, "id": "plc1_con"}
|
|
],
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [], "input_register": []
|
|
},
|
|
"monitors": [{
|
|
"outbound_connection_id": "plc1_con",
|
|
"id": "tank_level",
|
|
"value_type": "coil", # Wrong! Should be input_register
|
|
"slave_id": 1,
|
|
"address": 1,
|
|
"count": 1,
|
|
"interval": 0.5
|
|
}],
|
|
"controllers": []
|
|
}],
|
|
"plcs": [{
|
|
"name": "plc1",
|
|
"logic": "plc1.py",
|
|
"network": {"ip": "192.168.0.21", "docker_network": "vlan1"},
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [],
|
|
"input_register": [
|
|
{"address": 1, "count": 1, "io": "input", "id": "tank_level"}
|
|
]
|
|
}
|
|
}],
|
|
"sensors": [], "actuators": [], "hils": [],
|
|
"serial_networks": [], "ip_networks": []
|
|
}
|
|
config = Config.model_validate(raw)
|
|
errors = validate_hmi_semantics(config)
|
|
|
|
assert len(errors) >= 1
|
|
assert "value_type mismatch" in str(errors[0]).lower()
|
|
|
|
def test_address_mismatch_detected(self):
|
|
"""Monitor with wrong address should error."""
|
|
raw = {
|
|
"ui": {"network": {"ip": "192.168.0.1", "port": 8501, "docker_network": "vlan1"}},
|
|
"hmis": [{
|
|
"name": "hmi1",
|
|
"network": {"ip": "192.168.0.31", "docker_network": "vlan1"},
|
|
"inbound_connections": [],
|
|
"outbound_connections": [
|
|
{"type": "tcp", "ip": "192.168.0.21", "port": 502, "id": "plc1_con"}
|
|
],
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [], "input_register": []
|
|
},
|
|
"monitors": [{
|
|
"outbound_connection_id": "plc1_con",
|
|
"id": "tank_level",
|
|
"value_type": "input_register",
|
|
"slave_id": 1,
|
|
"address": 999, # Wrong address
|
|
"count": 1,
|
|
"interval": 0.5
|
|
}],
|
|
"controllers": []
|
|
}],
|
|
"plcs": [{
|
|
"name": "plc1",
|
|
"logic": "plc1.py",
|
|
"network": {"ip": "192.168.0.21", "docker_network": "vlan1"},
|
|
"registers": {
|
|
"coil": [], "discrete_input": [],
|
|
"holding_register": [],
|
|
"input_register": [
|
|
{"address": 1, "count": 1, "io": "input", "id": "tank_level"}
|
|
]
|
|
}
|
|
}],
|
|
"sensors": [], "actuators": [], "hils": [],
|
|
"serial_networks": [], "ip_networks": []
|
|
}
|
|
config = Config.model_validate(raw)
|
|
errors = validate_hmi_semantics(config)
|
|
|
|
assert len(errors) >= 1
|
|
assert "address mismatch" in str(errors[0]).lower()
|