189 lines
9.5 KiB
Python
189 lines
9.5 KiB
Python
from pathlib import Path
|
|
|
|
from unittest.mock import patch, PropertyMock
|
|
import pytest
|
|
import re
|
|
from rspec_tools.errors import RuleValidationError
|
|
from copy import deepcopy
|
|
|
|
from rspec_tools.rules import LanguageSpecificRule, RulesRepository
|
|
from rspec_tools.validation.metadata import validate_rule_specialization_metadata, validate_rule_metadata
|
|
|
|
@pytest.fixture
|
|
def rule_language(mockrules: Path):
|
|
rule = RulesRepository(rules_path=mockrules).get_rule('S100')
|
|
return rule.get_language('kotlin')
|
|
|
|
|
|
@pytest.fixture
|
|
def invalid_rules():
|
|
invalid_rules_path = Path(__file__).parent.parent.joinpath('resources', 'invalid-rules')
|
|
return RulesRepository(rules_path=invalid_rules_path)
|
|
|
|
|
|
def test_valid_metadata_passes_validation(rule_language: LanguageSpecificRule):
|
|
'''Check that language metadata are correctly overridden.'''
|
|
validate_rule_specialization_metadata(rule_language)
|
|
|
|
|
|
@patch('rspec_tools.validation.metadata.RULES_WITH_NO_LANGUAGES', [])
|
|
def test_rule_with_no_language(invalid_rules: RulesRepository):
|
|
s501 = invalid_rules.get_rule('S501')
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule S501 has no language-specific data'):
|
|
validate_rule_metadata(s501)
|
|
|
|
|
|
@patch('rspec_tools.validation.metadata.RULES_WITH_NO_LANGUAGES', ['S501'])
|
|
def test_rule_with_no_language_in_exception_list(invalid_rules: RulesRepository):
|
|
s501 = invalid_rules.get_rule('S501')
|
|
validate_rule_metadata(s501)
|
|
with patch.dict(s501.generic_metadata, [('status', 'deprecated')]):
|
|
validate_rule_metadata(s501)
|
|
|
|
|
|
@patch('rspec_tools.validation.metadata.RULES_WITH_NO_LANGUAGES', ['S501'])
|
|
def test_open_rule_with_no_language_in_exception_list(invalid_rules: RulesRepository):
|
|
s501 = invalid_rules.get_rule('S501')
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule S501 should be closed or deprecated'):
|
|
with patch.dict(s501.generic_metadata, [('status', 'ready')]):
|
|
validate_rule_metadata(s501)
|
|
|
|
|
|
@patch('rspec_tools.validation.metadata.RULES_WITH_NO_LANGUAGES', ['S120'])
|
|
def test_rule_expected_to_have_no_language(mockrules: Path):
|
|
valid_rule = RulesRepository(rules_path=mockrules).get_rule('S120')
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule S120 should have no specializations'):
|
|
validate_rule_metadata(valid_rule)
|
|
|
|
|
|
@patch('rspec_tools.validation.metadata.RULES_WITH_NO_LANGUAGES', [])
|
|
def test_rule_with_invalid_language(invalid_rules: RulesRepository):
|
|
s502 = invalid_rules.get_rule('S502')
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule S502 failed validation for these reasons:\n - Rule scala:S502 has invalid metadata'):
|
|
validate_rule_metadata(s502)
|
|
|
|
def test_rule_with_invalid_education_principles(invalid_rules: RulesRepository):
|
|
s503 = invalid_rules.get_rule('S503')
|
|
with pytest.raises(RuleValidationError, match=re.escape("Rule S503 failed validation for these reasons:\n - Rule scala:S503 has invalid metadata in 0: 'invalid' is not one of ['defense_in_depth', 'never_trust_user_input']")):
|
|
validate_rule_metadata(s503)
|
|
|
|
|
|
def test_rule_with_no_impacts(invalid_rules: RulesRepository):
|
|
s504 = invalid_rules.get_rule('S504')
|
|
with pytest.raises(RuleValidationError, match=re.escape("Rule S504 failed validation for these reasons:\n - Rule scala:S504 has invalid metadata in impacts: {} does not have enough properties")):
|
|
validate_rule_metadata(s504)
|
|
|
|
|
|
def test_rule_with_invalid_impacts(invalid_rules: RulesRepository):
|
|
s505 = invalid_rules.get_rule('S505')
|
|
with pytest.raises(RuleValidationError, match=re.escape("Rule S505 failed validation for these reasons:\n - Rule scala:S505 has invalid metadata in impacts: Additional properties are not allowed ('INVALID' was unexpected)")):
|
|
validate_rule_metadata(s505)
|
|
|
|
|
|
def test_rule_with_invalid_impact_level(invalid_rules: RulesRepository):
|
|
s506 = invalid_rules.get_rule('S506')
|
|
with pytest.raises(RuleValidationError, match=re.escape("Rule S506 failed validation for these reasons:\n - Rule scala:S506 has invalid metadata in MAINTAINABILITY: 'INVALID' is not one of ['INFO', 'LOW', 'MEDIUM', 'HIGH', 'BLOCKER']")):
|
|
validate_rule_metadata(s506)
|
|
|
|
|
|
def test_rule_with_invalid_attribute(invalid_rules: RulesRepository):
|
|
s507 = invalid_rules.get_rule('S507')
|
|
with pytest.raises(RuleValidationError, match=re.escape("Rule S507 failed validation for these reasons:\n - Rule scala:S507 has invalid metadata in attribute: 'INVALID' is not one of ['FORMATTED', 'CONVENTIONAL', 'IDENTIFIABLE', 'CLEAR', 'LOGICAL', 'COMPLETE', 'EFFICIENT', 'FOCUSED', 'DISTINCT', 'MODULAR', 'TESTED', 'LAWFUL', 'TRUSTWORTHY', 'RESPECTFUL']")):
|
|
validate_rule_metadata(s507)
|
|
|
|
|
|
def test_rule_with_unicode_in_metadata(invalid_rules: RulesRepository):
|
|
s4225 = invalid_rules.get_rule('S4225')
|
|
with pytest.raises(UnicodeDecodeError, match=fr'ascii'):
|
|
validate_rule_metadata(s4225)
|
|
|
|
def test_rule_that_is_fully_valid(mockrules: Path):
|
|
valid_rule = RulesRepository(rules_path=mockrules).get_rule('S120')
|
|
validate_rule_metadata(valid_rule)
|
|
|
|
|
|
def test_missing_required_property_fails_validation(rule_language: LanguageSpecificRule):
|
|
invalid_metadata = deepcopy(rule_language.metadata)
|
|
del invalid_metadata['title']
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule {rule_language.id} has invalid metadata'):
|
|
with patch.object(LanguageSpecificRule, 'metadata', new_callable=PropertyMock) as mock:
|
|
mock.return_value = invalid_metadata
|
|
validate_rule_specialization_metadata(rule_language)
|
|
|
|
|
|
def test_invalid_remediation_fails_validation(rule_language: LanguageSpecificRule):
|
|
invalid_metadata = deepcopy(rule_language.metadata)
|
|
invalid_metadata['remediation']["func"] = 42
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule {rule_language.id} has invalid metadata'):
|
|
with patch.object(LanguageSpecificRule, 'metadata', new_callable=PropertyMock) as mock:
|
|
mock.return_value = invalid_metadata
|
|
validate_rule_specialization_metadata(rule_language)
|
|
|
|
|
|
def test_adding_properties_fails_validation(rule_language: LanguageSpecificRule):
|
|
metadata = deepcopy(rule_language.metadata)
|
|
metadata['unknown'] = 42
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule {rule_language.id} has invalid metadata'):
|
|
with patch.object(LanguageSpecificRule, 'metadata', new_callable=PropertyMock) as mock:
|
|
mock.return_value = metadata
|
|
validate_rule_specialization_metadata(rule_language)
|
|
|
|
|
|
def test_ready_rule_with_replacement_fails_validation(rule_language: LanguageSpecificRule):
|
|
invalid_metadata = deepcopy(rule_language.metadata)
|
|
invalid_metadata['extra'] = { 'replacementRules': [ 'RSPEC-1234', 'RSPEC-5678' ]}
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule {rule_language.id} has invalid metadata: status'):
|
|
with patch.object(LanguageSpecificRule, 'metadata', new_callable=PropertyMock) as mock:
|
|
mock.return_value = invalid_metadata
|
|
validate_rule_specialization_metadata(rule_language)
|
|
|
|
|
|
def test_deprecated_rule_with_replacement_passes_validation(rule_language: LanguageSpecificRule):
|
|
metadata = deepcopy(rule_language.metadata)
|
|
metadata['extra'] = { 'replacementRules': [ 'RSPEC-1234' ]}
|
|
metadata['status'] = 'deprecated'
|
|
with patch.object(LanguageSpecificRule, 'metadata', new_callable=PropertyMock) as mock:
|
|
mock.return_value = metadata
|
|
validate_rule_specialization_metadata(rule_language)
|
|
|
|
|
|
def test_rule_with_incomplete_list_of_security_standard_fails_validation(rule_language: LanguageSpecificRule):
|
|
invalid_metadata = deepcopy(rule_language.metadata)
|
|
# "OWASP Top 10 2021", defined in the generic metadata is missing
|
|
invalid_metadata['securityStandards'] = {'ASVS 4.0': [], 'OWASP': [], 'CERT': []}
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule {rule_language.id} has invalid metadata: securityStandard'):
|
|
with patch.object(LanguageSpecificRule, 'metadata', new_callable=PropertyMock) as mock:
|
|
mock.return_value = invalid_metadata
|
|
validate_rule_specialization_metadata(rule_language)
|
|
|
|
|
|
def test_rule_with_complete_list_of_security_standard_passes_validation(rule_language: LanguageSpecificRule):
|
|
metadata = deepcopy(rule_language.metadata)
|
|
metadata['securityStandards'] = {'ASVS 4.0': [], 'OWASP': [], "OWASP Top 10 2021": []}
|
|
with patch.object(LanguageSpecificRule, 'metadata', new_callable=PropertyMock) as mock:
|
|
mock.return_value = metadata
|
|
validate_rule_specialization_metadata(rule_language)
|
|
|
|
def test_rule_with_invalid_format_for_security_standard_items_fails_validation(rule_language: LanguageSpecificRule):
|
|
invalid_security_standards_items = {
|
|
'OWASP': ['B1', 'AAA123', 'A0', ' A1', 'Not covered', ''],
|
|
'OWASP Top 10 2021': ['B1', 'AAA123', 'A0', ' A1', 'Not covered', ''],
|
|
'OWASP Mobile': ['B1', 'MMM123', 'M0', ' M1', 'Not covered', ''],
|
|
'PCI DSS 3.2': ['2.1.A', '2.1.1 ', 'Not covered', ''],
|
|
'PCI DSS 4.0': ['2.1.A', '2.1.1 ', 'Not covered', ''],
|
|
'CIS': ['2.1.A', '"2.1.1 ', 'Not covered', ''],
|
|
'HIPAA': ['Not covered', ''],
|
|
'CERT': ['MSC13-C', 'MSC13-C. ', 'Not covered', ''],
|
|
'MASVS': ['MSTG-CRYPTO-A', 'MSTG-CRYPTO-6 ', 'Not covered', ''],
|
|
'ASVS 4.0': ['A.1.2', ' 1.1.1', 'Not covered', '']
|
|
}
|
|
|
|
for security_standard in invalid_security_standards_items:
|
|
for item in invalid_security_standards_items[security_standard]:
|
|
invalid_metadata = deepcopy(rule_language.metadata)
|
|
invalid_metadata['securityStandards'] = { security_standard: [item] }
|
|
with pytest.raises(RuleValidationError, match=fr'^Rule {rule_language.id} has invalid metadata in 0: \'{item}\' does not match'):
|
|
with patch.object(LanguageSpecificRule, 'metadata', new_callable=PropertyMock) as mock:
|
|
mock.return_value = invalid_metadata
|
|
validate_rule_specialization_metadata(rule_language)
|