RULEAPI-574 Validate RSPEC description structure

generate_html.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euo pipefail
+
+mkdir -p out
+asciidoctor -R rules -D out 'rules/*/*/rule.adoc'
+cd rules
+find . -name 'metadata.json' -exec cp --parents '{}' ../out \;
+cd ..

rspec-tools/rspec_tools/cli.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 from typing import Optional
+from pathlib import Path
 
 import click
 from rspec_tools.checklinks import check_html_links
@@ -8,6 +9,7 @@ from rspec_tools.errors import InvalidArgumenError, RuleNotFoundError, RuleValid
 from rspec_tools.create_rule import RuleCreator, build_github_repository_url
 from rspec_tools.rules import RulesRepository
 from rspec_tools.validation.metadata import validate_metadata
+from rspec_tools.validation.description import validate_section_names
 
 @click.group()
 @click.option('--debug/--no-debug', default=False)
@@ -72,5 +74,26 @@ def validate_rules_metadata(rules):
     click.echo(message, err=True)
     raise click.Abort(message)
 
+@cli.command()
+@click.option('--d', required=True)
+@click.argument('rules', nargs=-1)
+def check_sections(d, rules):
+  '''Validate the section names.'''
+  out_dir = Path(__file__).parent.parent.joinpath(d)
+  rule_repository = RulesRepository(rules_path=out_dir)
+  error_counter = 0
+  for rule in rule_repository.rules:
+    if rules and rule.key not in rules:
+      continue
+    for lang_spec_rule in rule.specializations:
+      try:
+        validate_section_names(lang_spec_rule)
+      except RuleValidationError as e:
+        click.echo(e.message, err=True)
+        error_counter += 1
+  if error_counter > 0:
+    message = f"Validation failed due to {error_counter} errors"
+    click.echo(message, err=True)
+    raise click.Abort(message)
 
 __all__=['cli']

rspec-tools/rspec_tools/rules.py

@@ -2,14 +2,17 @@
 import json
 from pathlib import Path
 from typing import Final, Generator, Iterable, Optional
+from bs4 import BeautifulSoup
 
 
 METADATA_FILE_NAME: Final[str] = 'metadata.json'
+DESCRIPTION_FILE_NAME: Final[str] = 'rule.html'
 
 class LanguageSpecificRule:
   language_path: Final[Path]
   rule: 'GenericRule'
   __metadata: Optional[dict] = None
+  __description: Optional[object] = None
 
   def __init__(self, language_path: Path, rule: 'GenericRule'):
     self.language_path = language_path
@@ -32,6 +35,14 @@ class LanguageSpecificRule:
     self.__metadata = self.rule.generic_metadata | lang_metadata
     return self.__metadata
 
+  @property
+  def description(self):
+    if self.__description is not None:
+      return self.__description
+    description_path = self.language_path.joinpath(DESCRIPTION_FILE_NAME)
+    soup = BeautifulSoup(description_path.read_bytes(),features="html.parser")
+    self.__description = soup
+    return self.__description
 
 class GenericRule:
   rule_path: Final[Path]
@@ -74,4 +85,4 @@ class RulesRepository:
     return (GenericRule(child) for child in self.rules_path.glob('S*') if child.is_dir())
     
   def get_rule(self, ruleid: str):
-    return GenericRule(self.rules_path.joinpath(ruleid))
+    return GenericRule(self.rules_path.joinpath(ruleid))

rspec-tools/rspec_tools/validation/description.py

@@ -0,0 +1,43 @@
+import json
+from bs4 import BeautifulSoup
+from typing import Final
+
+from rspec_tools.errors import RuleValidationError
+from rspec_tools.rules import LanguageSpecificRule
+
+# The list of all the sections currently accepted by the script.
+# The list includes multiple variants for each title because they all occur
+# in the migrated RSPECs.
+# Further work required to shorten the list by renaming the sections in some RSPECS
+# to keep only on version for each title.
+ACCEPTED_SECTION_NAMES: Final[list[str]] = ['Noncompliant Code Example',
+                                            'Noncompliant Code Example.',
+                                            'Noncompliant Code Example:',
+                                            'Noncompliant Code Examples',
+                                            'NonCompliant Code Example',
+                                            'Compliant Solution',
+                                            'Compliant Solutions',
+                                            'Compliant solution',
+                                            'Compliant Solution:',
+                                            'Compliant Example',
+                                            'Compliant Code Example',
+                                            'Compliant',
+                                            'See',
+                                            'See:',
+                                            'See also',
+                                            'See Also',
+                                            'Exceptions',
+                                            'Sensitive Code Example',
+                                            'Sensitive Code Examples',
+                                            'Ask Yourself Whether',
+                                            'Recommended Secure Coding Practices',
+                                            'Deprecated']
+
+def validate_section_names(rule_language: LanguageSpecificRule):
+  descr = rule_language.description
+  for h2 in descr.findAll('h2'):
+    name = h2.text.strip()
+    if name not in ACCEPTED_SECTION_NAMES:
+      raise RuleValidationError(f'Rule {rule_language.id} has unconventional header "{name}"')
+
+__all__=['validate_metadata']

rspec-tools/tests/resources/rules/S100/cfamily/rule.html

@@ -0,0 +1,28 @@
+<body>
+<div class="paragraph">
+<p>Shared naming conventions allow teams to collaborate efficiently. This rule checks that all function names match a provided regular expression.</p>
+</div>
+<div class="sect1">
+<h2 id="_noncompliant_code_example">Noncompliant Code Example</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>With default provided regular expression: <code>^[a-z][a-zA-Z0-9]*$</code>:</p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre>void DoSomething (void);</pre>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_compliant_solution">Compliant Solution</h2>
+<div class="sectionbody">
+<div class="listingblock">
+<div class="content">
+<pre>void doSomething (void);</pre>
+</div>
+</div>
+</div>
+</div>
+</body>

rspec-tools/tests/validation/test_description_validation.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+from unittest.mock import patch, PropertyMock
+import pytest
+from rspec_tools.errors import RuleValidationError
+from copy import deepcopy
+
+from rspec_tools.rules import LanguageSpecificRule, RulesRepository
+from rspec_tools.validation.description import validate_section_names
+
+@pytest.fixture
+def rule_language(mockrules: Path):
+  rule = RulesRepository(rules_path=mockrules).get_rule('S100')
+  return rule.get_language('cfamily')
+
+def test_valid_sections_passes_validation(rule_language: LanguageSpecificRule):
+  '''Check that description with standard sections is considered valid.'''
+  validate_section_names(rule_language)
+
+def test_unexpected_section_fails_validation(rule_language: LanguageSpecificRule):
+  '''Check that unconventional section header breaks validation.'''
+  invalid_description = deepcopy(rule_language.description)
+  invalid_header = invalid_description.new_tag('h2')
+  invalid_header.string = 'Invalid header'
+  invalid_description.body.insert(1, invalid_header)
+  with pytest.raises(RuleValidationError, match=fr'^Rule {rule_language.id} has unconventional header "Invalid header"'):
+    with patch.object(LanguageSpecificRule, 'description', new_callable=PropertyMock) as mock:
+      mock.return_value = invalid_description
+      validate_section_names(rule_language)

validate_asciidoc.sh

@@ -1,3 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+
+./generate_html.sh
+
+#validate sections in asciidoc
+cd rspec-tools
+pipenv install -e .
+pipenv run rspec-tools check-sections --d ../out
+cd ..
+
 for dir in rules/*
 do
   dir=${dir%*/}

validate_links.sh

@@ -1,12 +1,9 @@
 #!/bin/bash
 set -euo pipefail
 
+./generate_html.sh
+
 #validate links in asciidoc
-mkdir -p out
-asciidoctor -R rules -D out 'rules/*/*/rule.adoc'
-cd rules
-  find . -name 'metadata.json' -exec cp --parents '{}' ../out \;
-cd ..
 cd rspec-tools
 pipenv install -e .
 pipenv run rspec-tools check-links --d ../out