Add CI task to test CI tests (#2768)

Extract the bits using the custom asciidoctor to a dedicated file.
This new script uses `set -e` to catch unexpected errors.
It is also covered with integration tests that get executed on the CI.

This serves as the groundwork to integrate more advanced validation
tests.

.cirrus.yml

@@ -86,6 +86,16 @@ validate_metadata_task:
   metadata_tests_script:
     - ./ci/validate_metadata.sh
 
+validate_ci_tests_task:
+  skip: "!changesInclude('ci_tests/*', 'ci/*')"
+  eks_container:
+    <<: *CONTAINER_DEFINITION
+    dockerfile: ci/Dockerfile
+    cpu: 1
+    memory: 2G
+  ci_tests_script:
+    - ./ci_tests/asciidoc_validation/run_tests.sh
+
 validate_asciidoc_task:
   eks_container:
     <<: *CONTAINER_DEFINITION
@@ -121,6 +131,7 @@ all_required_checks_task:
     - frontend_tests
     - validate_metadata
     - validate_asciidoc
+    - validate_ci_tests
   eks_container:
     <<: *CONTAINER_DEFINITION
     dockerfile: ci/Dockerfile

ci/asciidoc_validation/custom-asciidoctor

@@ -62,7 +62,7 @@ class IncludeLogger < Asciidoctor::Extensions::IncludeProcessor
 
     rule_dir = rule_dir + '/' # Don't allow S100 to include things from S1000.
     if !include_path.start_with?(rule_dir) && !include_path.start_with?(shared_dir)
-      logger.info("ASCIIDOC LOGGER CROSSREFERENCE:#{rule_id} crossreferences #{include_path}")
+      logger.info("ASCIIDOC LOGGER CROSSREFERENCE:#{rule_id} cross-references #{include_path}")
     end
 
     logger.info("ASCIIDOC LOGGER INCLUDE:#{include_path}")

ci/asciidoc_validation/validate.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+#
+# Validate file inclusion and cross-references.
+#
+# This script is parametrized by this environment variable:
+#  * TOPLEVEL: $TOPLEVEL/rules and $TOPLEVEL/shared_content will be validated.
+#
+# The script exists with a non-zero value when an unexpected error happened or
+# there are asciidoc validation errors.
+#
+# Validation errors are reported on stderr.
+#
+# Implementation details:
+#
+#   The validation of file inclusion and cross-references needs to be done
+#   on all rule descriptions, including the default, language-agnostic
+#   description, with rspecator-view. Otherwise, a rule could drop an include
+#   of a shared_content asciidoc and that file could become unused.
+#
+#   We use a custom asciidoctor with extra logging for this purpose.
+#   The format for the interesting log entries are:
+#     asciidoctor: INFO: ASCIIDOC LOGGER MAIN FILE: $PATH
+#     asciidoctor: INFO: ASCIIDOC LOGGER INCLUDED: $PATH
+#     asciidoctor: INFO: ASCIIDOC LOGGER CROSSREFERENCE: $RULEID cross-references $PATH
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
+TOPLEVEL="$(realpath "${TOPLEVEL}")"
+RULES_DIR="${TOPLEVEL}/rules"
+SHARED_CONTENT_DIR="${TOPLEVEL}/shared_content"
+TMPOUT_DIR="$(mktemp -d)"
+
+exit_code=0
+
+grep_nofail() {
+  # Grep but always exit with 0 when there are no matches.
+  # Yet, exit with non-zero if an error occured.
+  grep "$@" || [ "$?" == "1" ]
+}
+
+find "${RULES_DIR}" -name 'rule.adoc' \
+  | xargs "${SCRIPT_DIR}/custom-asciidoctor" -a rspecator-view --verbose -R "${RULES_DIR}" -D "${TMPOUT_DIR}" 2>&1 \
+  | grep_nofail -e 'ASCIIDOC LOGGER' \
+  > "${TMPOUT_DIR}/asciidoc_introspection"
+
+grep -ve 'CROSSREFERENCE' "${TMPOUT_DIR}/asciidoc_introspection" \
+  | cut -d ':' -f 4 \
+  | sort -u \
+  > "${TMPOUT_DIR}/used_asciidoc_files"
+
+git ls-files --cached -- "${RULES_DIR}/**.adoc" "${SHARED_CONTENT_DIR}/**.adoc" \
+  | xargs realpath \
+  > "${TMPOUT_DIR}/all_asciidoc_files"
+
+cross_references=$(grep_nofail -e 'CROSSREFERENCE' "${TMPOUT_DIR}/asciidoc_introspection" | cut -d ':' -f 4 | sort -u)
+if [[ -n "$cross_references" ]]; then
+  echo >&2 'ERROR: Some rules try to include content from unallowed directories.'
+  echo >&2 'To share content between rules, you should use the "shared_content" folder at the root of the repository.'
+  echo >&2 'List of errors:'
+  echo >&2 "${cross_references}"
+  exit_code=1
+fi
+
+orphans=$(comm -1 -3 <(sort -u "${TMPOUT_DIR}/used_asciidoc_files") <(sort -u "${TMPOUT_DIR}/all_asciidoc_files"))
+if [[ -n "$orphans" ]]; then
+  printf >&2 'ERROR: These adoc files are not included anywhere:\n-----\n%s\n-----\n' "$orphans"
+  exit_code=1
+fi
+
+exit $exit_code

ci/validate_asciidoc.sh

@@ -133,47 +133,11 @@ else
 fi
 find rules -name "tmp*.adoc" -delete
 
-# Cover file inclusion and crossreferences.
+# Validate file inclusion, cross-references, and other properties.
 #
-# This needs to be done on all rule descriptions, including the default,
-# language-agnostic description, with rspecator-view. Otherwise, a rule
-# could drop an include of a shared_content asciidoc and that file could
-# become unused.
-#
-# We use a custom asciidoctor with extra logging for this purpose.
-# The format for the interesting log entries are:
-#   asciidoctor: INFO: ASCIIDOC LOGGER MAIN FILE: $PATH
-#   asciidoctor: INFO: ASCIIDOC LOGGER INCLUDED: $PATH
-#   asciidoctor: INFO: ASCIIDOC LOGGER CROSSREFERENCE: $RULEID crossreferences $PATH
-outdir="$(mktemp -d)"
-find rules -name 'rule.adoc' \
-  | xargs ./ci/custom-asciidoctor -a rspecator-view --verbose -R rules -D "${outdir}" 2>&1 \
-  | grep -e 'ASCIIDOC LOGGER' \
-  > asciidoc_introspection
-
-grep -ve 'CROSSREFERENCE' asciidoc_introspection \
-  | cut -d ':' -f 4 \
-  | sort -u \
-  > used_asciidoc_files
-
-git ls-files --cached -- 'rules/**.adoc' 'shared_content/**.adoc' \
-  | xargs realpath \
-  > all_asciidoc_files
-
-cross_references=$(grep -e 'CROSSREFERENCE' asciidoc_introspection | cut -d ':' -f 4 | sort -u)
-if [[ -n "$cross_references" ]]; then
-  echo 'ERROR: Some rules try to include content from unallowed directories.'
-  echo 'To share content between rules, you should use the "shared_content" folder at the root of the repository.'
-  echo 'List of errors:'
-  echo "${cross_references}"
-  exit_code=1
-fi
-
-orphans=$(comm -1 -3 <(sort -u used_asciidoc_files) <(sort -u all_asciidoc_files))
-if [[ -n "$orphans" ]]; then
-  printf 'ERROR: These adoc files are not included anywhere:\n-----\n%s\n-----\n' "$orphans"
-  exit_code=1
-fi
+# This part of the validation is extracted in a separate script,
+# which is covered by tests unlike what is above this line.
+TOPLEVEL=. ./ci/asciidoc_validation/validate.sh || exit_code=1
 
 if (( exit_code == 0 )); then
   echo "Success"

ci_tests/asciidoc_validation/run_tests.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+#
+# Run integration tests for ci/asciidoc_validation.
+
+set -uo pipefail
+
+# We could write complex checks to ensure only specific commands fail and emit
+# a specific error message. Instead, we rely on `set -xe` to consistently and
+# reliably exit with non-zero if any command fails and pinpoint which command
+# failed in the trace output. We also use a trap on ERR to give users a short
+# hint, and `set -E` to propagate this trap to shell functions and subshells.
+#
+# This allows use to write tests as simple commands, such as
+#   test -f file_exists
+#
+# When we want to ensure a command fails, we use this pattern:
+#   { ! command; }
+set -xeE
+err_trap() {
+  set +x # Disable tracing when displaying stackframe.
+  echo "Some test failed; look at the trace for more. Here is the stackframe:" >&2
+  i=0
+  while caller $i >&2
+  do
+    (( i++ )) || :
+  done
+}
+trap err_trap ERR
+
+# Ensure the script we test exists and is executable.
+GIT_TOPLEVEL_DIR="$(git rev-parse --show-toplevel)"
+VALIDATE_SCRIPT="${GIT_TOPLEVEL_DIR}/ci/asciidoc_validation/validate.sh"
+test -f "${VALIDATE_SCRIPT}"
+test -x "${VALIDATE_SCRIPT}"
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
+
+run_test() {
+  # Run validation script on $1.
+  # Ensure the output contains $2...$N.
+  tmp="$(mktemp -d)"
+  stderr_log="${tmp}/stderr_log"
+  stdout_log="${tmp}/stdout_log"
+  if TOPLEVEL="${SCRIPT_DIR}/$1" "${VALIDATE_SCRIPT}" 2> "${stderr_log}" > "${stdout_log}"
+  then
+    # The validation succeeded. We expect nothing in the output.
+    [ $# -eq 1 ] # no tests
+    test -f "${stderr_log}"
+    { ! test -s "${stderr_log}"; }
+  else
+    # We expect at least on check of the stderr content.
+    [ $# -gt 1 ]
+    shift
+
+    for query in "$@"
+    do
+      test -n "${query}"
+      grep -q -e "${query}" "${stderr_log}"
+    done
+  fi
+
+  # Regardless of success or failure, the stdout is expected to be empty.
+  test -f "${stdout_log}"
+  { ! test -s "${stdout_log}"; }
+}
+
+run_test "test_valid"
+
+run_test "test_unused_adoc" \
+  "ERROR: These adoc files are not included anywhere:" \
+  "rules/S100/java/unused.adoc" \
+  "rules/S100/unused.adoc" \
+  "shared_content/unused.adoc"
+
+run_test "test_bad_cross_ref" \
+  "ERROR: Some rules try to include content from unallowed directories." \
+  "S100 cross-references .*rules/S1000/bad.adoc" \
+  "S1000 cross-references .*rules/S100/java/bad.adoc"
+
+echo "All tests passed"

ci_tests/asciidoc_validation/test_bad_cross_ref/rules/S100/cfamily/rule.adoc

@@ -0,0 +1,3 @@
+Bad include from S100
+
+include::../../S1000/bad.adoc[]

ci_tests/asciidoc_validation/test_bad_cross_ref/rules/S100/java/bad.adoc

@@ -0,0 +1 @@
+java S100

ci_tests/asciidoc_validation/test_bad_cross_ref/rules/S100/java/rule.adoc

@@ -0,0 +1,3 @@
+This include is fine
+
+include::./bad.adoc[]

ci_tests/asciidoc_validation/test_bad_cross_ref/rules/S1000/bad.adoc

@@ -0,0 +1 @@
+File from S1000

ci_tests/asciidoc_validation/test_bad_cross_ref/rules/S1000/java/rule.adoc

@@ -0,0 +1 @@
+include::../../../shared_content/java/shared.adoc[]

ci_tests/asciidoc_validation/test_bad_cross_ref/shared_content/java/shared.adoc

@@ -0,0 +1 @@
+include::../../rules/S100/java/bad.adoc[]

ci_tests/asciidoc_validation/test_unused_adoc/rules/S100/cfamily/rule.adoc

@@ -0,0 +1,3 @@
+I am a main file.
+
+include::./used.adoc[]

ci_tests/asciidoc_validation/test_unused_adoc/rules/S100/cfamily/used.adoc

@@ -0,0 +1 @@
+I am included.

ci_tests/asciidoc_validation/test_unused_adoc/rules/S100/java/rule.adoc

@@ -0,0 +1,3 @@
+I am a main file.
+
+include::../../../shared_content/java/included.adoc[]

ci_tests/asciidoc_validation/test_unused_adoc/shared_content/java/included.adoc

@@ -0,0 +1 @@
+I got included.

ci_tests/asciidoc_validation/test_valid/rules/S100/cfamily/another.adoc

@@ -0,0 +1 @@
+another

ci_tests/asciidoc_validation/test_valid/rules/S100/cfamily/rule.adoc

@@ -0,0 +1,5 @@
+S100 cfamily
+
+include::./another.adoc[]
+
+include::../../../shared_content/anything.adoc[]

ci_tests/asciidoc_validation/test_valid/rules/S100/rule.adoc

@@ -0,0 +1,3 @@
+generic description
+
+include::../../shared_content/anything.adoc[]

ci_tests/asciidoc_validation/test_valid/rules/S200/java/rule.adoc

@@ -0,0 +1,3 @@
+S200 java
+
+include::../../../shared_content/anything.adoc[]

ci_tests/asciidoc_validation/test_valid/shared_content/anything.adoc

@@ -0,0 +1 @@
+anything