diff --git a/tests/roles/custom-test-fonts/README.md b/tests/roles/custom-test-fonts/README.md new file mode 100644 index 0000000..d28fc26 --- /dev/null +++ b/tests/roles/custom-test-fonts/README.md @@ -0,0 +1,69 @@ +# Ansible role for tests using fontconfig + +Put this role in your `tests.yml` playbook. The playbook will first install +package dependencies listed on playbook on your localhost, then it will proceed +to run testing. You can redefine the following variables: + + * **artifacts**: An artifacts directory on localhost to store logs + * **remote_artifacts**: The directory on the system under test where the logs + are stored. Note: if this variable is left undefined, it will default to + `/tmp/artifacts` + * **required_packages**: A list of prerequisite packages required by tests. + Please note that for Atomic Host, additional packages will be installed + using the `rpm-ostree` command which is affecting the test subject (it's + similar as rebuilding an rpm package to be tested) so this should be used + with caution on only when necessary. + * **path_prefix**: The directory on the system where fonts are installed. + please use one in coverage sub-parameter if having different path_prefix + per sub-packages. + * **package**: The package name to test. this is used to find out fontconfig + config file. please use one in families sub-parameter if having different + config files per sub-packages. + * **coverage**: A list of languages for language coverage tests. + * **families**: A list of family test cases. + +## Language coverage test parameters + +Supporting two types of formats. one is a simple list of languages: + + coverage: + - en + - fr + +Another one is a dictionary that has a language as a key and values as parameters: + + coverage: + en: + ... + fr: + ... + +You can redefine the following variables for dictionary format: + + * **exclude**: A list of font file names to exclude on this testing. this is + useful to avoid unexpected failures on iterating tests when a package has + multiple font files and has different coverages but you need to prevent + testing for few fonts which has different coverages to them. + Please note that the file name is relative to `path_prefix` parameter. also + good to consider using `include` if non-targeted files is more than targeted. + * **include**: A list of font file names to include on this testing. this is + useful to avoid unexpected failures on iterating tests when a pcakge has + multiple font files and has different coverages but you need to prevent + testing for few fonts which has different coverages to them. + Please note that the file name is relative to `path_prefix` parameter. also + good to consider using `exclude` if targeted files is more than non-targeted. + * **name**: The name to store logs. the test script is trying to make an unique + file names to store logs but not perfectly working in some cases. this is + optional parameter to make it unique by yourself. + * **path_prefix**: A list of directory names where fonts are installed on system. + this is optional parameter and tries to obtain the font paths from installed + packages by `required_packages` if not available. + +## Family test parameters + + * **lang**: A language to test family name for. + * **alias**: An alias name to test. + * **family**: A family name to test, which is supposed to be assinged to the alias. + * **package**: The package name to test. this is used to find out fontconfig + config file. this is optional. if not specified here, global `package` + parameter will be used. diff --git a/tests/roles/custom-test-fonts/defaults/main.yml b/tests/roles/custom-test-fonts/defaults/main.yml new file mode 100644 index 0000000..fe09145 --- /dev/null +++ b/tests/roles/custom-test-fonts/defaults/main.yml @@ -0,0 +1,7 @@ +--- + +role_pkgs_req: + - fontconfig + - fontconfig-devel + - pkg-config + - rsync diff --git a/tests/roles/custom-test-fonts/files/run-family-test b/tests/roles/custom-test-fonts/files/run-family-test new file mode 100755 index 0000000..bbbed73 --- /dev/null +++ b/tests/roles/custom-test-fonts/files/run-family-test @@ -0,0 +1,162 @@ +#! /bin/bash -efu + +debug() { + if [ -n "$DEBUG" ]; then + echo "$*" >&2 + fi +} + +msg_usage() { + cat <<_EOF_ +Run family test. + +Usage: +$PROG + +Options: +-h, --help Display this help and exit +-v, --verbose Turn on debug +-l, --lang=LANG Test LANG language coverage (default: en) +-f, --family=FILE Set a family name supposed to be assigned for alias. +-g, --alias=STR Set an alias name. (default: sans-serif) +-a, --artifactsdir=DIR Set environment dir to store artifacts +-k, --package=NAME Set a package name for fonts. +_EOF_ +} + +PROG="${PROG:-${0##*/}}" +DEBUG="${DEBUG:-}" +OPT_LANG="${OPT_LANG:-en}" +OPT_FAMILY="${OPT_FAMILY:-}" +OPT_ARTIFACTS_DIR="${OPT_ARTIFACTS_DIR:-}" +OPT_ALIAS="${OPT_ALIAS:-sans-serif}" +OPT_PACKAGE="${OPT_PACKAGE:-}" + +opt=$(getopt -n "$0" --options "hvl:f:t:a:g:k:" --longoptions "help,verbose,lang:,family:,test:,artifactsdir:,alias:,package:" -- "$@") +eval set -- "$opt" +while [[ $# -gt 0 ]]; do + case "$1" in + -k|--package) + OPT_PACKAGE="$2" + shift 2 + ;; + -g|--alias) + OPT_ALIAS="$2" + shift 2 + ;; + -a|--artifactsdir) + OPT_ARTIFACTS_DIR="$2" + shift 2 + ;; + -f|--family) + OPT_FAMILY="$2" + shift 2 + ;; + -l|--lang) + OPT_LANG="$2" + shift 2 + ;; + -v|--verbose) + DEBUG="-v" + shift + ;; + -h|--help) + msg_usage + exit 0 + ;; + --) + shift + ;; + *) + msg_usage + exit 1 + esac +done + +if [ -z "$OPT_ARTIFACTS_DIR" ] || [ -z "$OPT_LANG" ] || [ -z "$OPT_FAMILY" ]; then + echo "Use: $PROG -h for help." + exit 0 +fi + +debug "Alias: $OPT_ALIAS" +debug "Family: $OPT_FAMILY" +debug "Lang: $OPT_LANG" +debug "Artifacts dir: $OPT_ARTIFACTS_DIR" +debug "Package name: $OPT_PACKAGE" +STR_TEST_DASHED=$(echo "${OPT_PACKAGE}_${OPT_ALIAS}_${OPT_LANG}" | sed -e 's/\//-/g' -e 's/ /-/g') +debug "Log file: $STR_TEST_DASHED.log" + +clean_exit() { + rc=$?; + trap - SIGINT SIGTERM SIGABRT EXIT + echo "Run test $OPT_ALIAS: done. Test's exit code: $rc" + for pid in $(ps -o pid --no-headers --ppid $$); do + if [ -n "$(ps -p $pid -o pid=)" ]; then + kill -s HUP $pid + fi + done + local log_file_name="$STR_TEST_DASHED.log" + local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name" + local status + if [[ $rc -eq 127 ]]; then + status="ERROR" + elif grep -q "RESULT: WARN" "$log_file_path"; then + status="ERROR" + elif grep -q "RESULT: FAIL" "$log_file_path"; then + status="FAIL" + elif grep -q "RESULT: PASS" "$log_file_path"; then + status="PASS" + elif grep -q "FAIL" "$log_file_path"; then + status="FAIL" + elif grep -q "PASS" "$log_file_path"; then + status="PASS" + else + status="ERROR" + fi + echo "$status $OPT_ALIAS" >> "$OPT_ARTIFACTS_DIR/test.log" + mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}" + local results="$OPT_ARTIFACTS_DIR/results.yml" + local result=$(echo $status | tr '[:upper:]' '[:lower:]') + test -f "$results" || echo 'results:' > "$results" + printf '%s\n' '' \ + "- test: $OPT_ALIAS" \ + " result: $result" \ + " logs:" \ + " - ${status}_${log_file_name}" \ + >> "$results" + exit 0 +} +trap clean_exit SIGINT SIGTERM SIGABRT EXIT + +cachedir=`pkg-config --variable cachedir fontconfig` +tmpconfd=`mktemp --tmpdir -d fontsci.XXXXXXXX` +conf=$(for i in `rpm -ql $OPT_PACKAGE | grep conf.d`; do + echo "$i" +done) +cat <<_EOF_> $tmpconfd/fonts.conf + + /usr/share/fonts + $conf + $cachedir + +_EOF_ +debug "Config: `cat $tmpconfd/fonts.conf`" + +mkdir -p "$OPT_ARTIFACTS_DIR" +export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log" +logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log" +logfile="$(realpath "$logfile")" +exec > >(tee -a "$logfile") 2>&1 + +debug "Check family assignment" +res=`FONTCONFIG_FILE=$tmpconfd/fonts.conf fc-match -f "%{family[0]}" :family=$OPT_ALIAS:lang=$OPT_LANG` +ret=0 +if [ "x$res" = "x$OPT_FAMILY" ]; then + echo "RESULT: PASS: $OPT_FAMILY was assigned to $OPT_ALIAS as expected" +else + echo "RESULT: FAIL: $OPT_FAMILY wasn't assigned to $OPT_ALIAS (actual result: $res)" + ret=1 +fi +rm -rf $tmpconfd + +exit $ret diff --git a/tests/roles/custom-test-fonts/files/run-lang-coverage-test b/tests/roles/custom-test-fonts/files/run-lang-coverage-test new file mode 100755 index 0000000..a0de1fb --- /dev/null +++ b/tests/roles/custom-test-fonts/files/run-lang-coverage-test @@ -0,0 +1,252 @@ +#! /bin/bash -efu + +debug() { + if [ -n "$DEBUG" ]; then + echo "$*" >&2 + fi +} + +msg_usage() { + cat <<_EOF_ +Run language coverage test. + +Usage: +$PROG + +Options: +-h, --help Display this help and exit +-v, --verbose Turn on debug +-l, --lang=LANG Test LANG language coverage (default: en) +-p, --path=PATH Test fonts on PATH +-k, --package=PACKAGE Specify PACKAGE to obtain some information +-n, --name=NAME Set NAME to store a log file. +-a, --artifactsdir=DIR test environment dir to store artifacts +-e, --exclude=FILE Exclude FILE to check. +-i, --include=FILE Include File to check. +_EOF_ +} + +PROG="${PROG:-${0##*/}}" +DEBUG="${DEBUG:-}" +OPT_LANG="${OPT_LANG:-en}" +OPT_PATH=() +OPT_PACKAGE=() +OPT_ARTIFACTS_DIR="${OPT_ARTIFACTS_DIR:-}" +OPT_EXCLUDE=() +OPT_INCLUDE=() +OPT_NAME="${OPT_NAME:-}" + +opt=$(getopt -n "$0" --options "hvl:p:k:n:a:e:i:" --longoptions "help,verbose,lang:,path:,package:,name:,artifactsdir:,exclude:,include:" -- "$@") +eval set -- "$opt" +while [[ $# -gt 0 ]]; do + case "$1" in + -n|--name) + OPT_NAME="$2" + shift 2 + ;; + -i|--include) + OPT_INCLUDE+=("$2") + shift 2 + ;; + -e|--exclude) + OPT_EXCLUDE+=("$2") + shift 2 + ;; + -a|--artifactsdir) + OPT_ARTIFACTS_DIR="$2" + shift 2 + ;; + -p|--path) + OPT_PATH+=("$2") + shift 2 + ;; + -k|--package) + OPT_PACKAGE+=("$2") + shift 2 + ;; + -l|--lang) + OPT_LANG="$2" + shift 2 + ;; + -v|--verbose) + DEBUG="-v" + shift + ;; + -h|--help) + msg_usage + exit 0 + ;; + --) + shift + ;; + *) + msg_usage + exit 1 + esac +done + +if [ -z "$OPT_ARTIFACTS_DIR" ] || [ -z "$OPT_LANG" ] || [ ! -v OPT_PATH ] && [ ! -v OPT_PACKAGE ]; then + echo "Use: $PROG -h for help." + exit 0 +fi + +STR_TEST_DASHED=$(echo "${OPT_NAME:-$OPT_LANG}" | sed -e 's/\//-/g') + +clean_exit() { + rc=$?; + trap - SIGINT SIGTERM SIGABRT EXIT + echo "Run test $OPT_LANG: done. Test's exit code: $rc" + for pid in $(ps -o pid --no-headers --ppid $$); do + if [ -n "$(ps -p $pid -o pid=)" ]; then + kill -s HUP $pid + fi + done + local log_file_name="$STR_TEST_DASHED.log" + local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name" + local status + if [[ $rc -eq 127 ]]; then + status="ERROR" + elif grep -q "RESULT: WARN" "$log_file_path"; then + status="ERROR" + elif grep -q "RESULT: FAIL" "$log_file_path"; then + status="FAIL" + elif grep -q "RESULT: PASS" "$log_file_path"; then + status="PASS" + elif grep -q "WARN" "$log_file_path"; then + status="ERROR" + elif grep -q "FAIL" "$log_file_path"; then + status="FAIL" + elif grep -q "PASS" "$log_file_path"; then + status="PASS" + else + status="ERROR" + fi + echo "$status $OPT_LANG" >> "$OPT_ARTIFACTS_DIR/test.log" + mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}" + local results="$OPT_ARTIFACTS_DIR/results.yml" + local result=$(echo $status | tr '[:upper:]' '[:lower:]') + test -f "$results" || echo 'results:' > "$results" + printf '%s\n' '' \ + "- test: $OPT_LANG" \ + " result: $result" \ + " logs:" \ + " - ${status}_${log_file_name}" \ + >> "$results" + exit 0 +} +trap clean_exit SIGINT SIGTERM SIGABRT EXIT + +mkdir -p "$OPT_ARTIFACTS_DIR" +export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log" +logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log" +logfile="$(realpath "$logfile")" +exec > >(tee -a "$logfile") 2>&1 + +for pkg in ${OPT_PACKAGE[@]}; do + if ! rpm -q ${pkg} > /dev/null 2>&1; then + echo "Package isn't installed or maybe a typo: ${pkg}" + exit 127 + fi + d=$(for d in $(rpm -ql ${pkg}|grep /usr/share/fonts); do + dirname $d + done | sort | uniq | grep /usr/share/fonts/) + if [[ ! " ${OPT_PATH[@]} " =~ " ${d} " ]]; then + OPT_PATH+=("$d") + fi +done + +expand_regex() { + local p ret=() + local regex="$1" + shift + debug "Expanding $regex" + for p; do + set +f + debug "$p: $regex" + (cd $p; + local x=$(find -regextype posix-egrep -regex "./$regex" -print|sed -e 's,^\./,,g') + debug "$x" + ret+=($x) + set -f + echo -n ${ret[@]} + ) + done + echo -n ${ret[@]} +} + +iv=() +ev=() +x=() +for p in ${OPT_INCLUDE[@]}; do + x=$(expand_regex $p ${OPT_PATH[@]}) + if [ "x$x" == "x" ]; then + echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?" + continue + fi + iv=("${iv[@]}" "${x[@]}") +done +for p in ${OPT_EXCLUDE[@]}; do + x=$(expand_regex $p ${OPT_PATH[@]}) + if [ "x$x" == "x" ]; then + echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?" + continue + fi + ev=("${ev[@]}" "${x[@]}") +done +OPT_EXCLUDE=(${ev[@]}) +OPT_INCLUDE=(${iv[@]}) + +debug "Path: ${OPT_PATH[@]}" +debug "Lang: $OPT_LANG" +debug "Artifacts dir: $OPT_ARTIFACTS_DIR" +debug "Exclude: ${#OPT_EXCLUDE[@]}: ${OPT_EXCLUDE[@]}" +debug "Include: ${#OPT_INCLUDE[@]}: ${OPT_INCLUDE[@]}" + +contains() { + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 1; done + return 0 +} + +debug "Check language coverage" +ret=0 +set +f +for p in ${OPT_PATH[@]}; do + for i in `find $p -regex '.*/*\.\(t1\)?\(ttf\)?\(otf\)?\(ttc\)?\(pcf.*\)?\(pfa\)?'`; do + set -f + debug "$i" + if test -f $i; then + n=`basename $i` + set +e + contains "$n" "${OPT_EXCLUDE[@]}" + r=$? + set -e + if [ $r -eq 1 ]; then + debug "ignoring $i" + continue + fi + if [ ${#OPT_INCLUDE[@]} -ne 0 ]; then + set +e + contains "$n" "${OPT_INCLUDE[@]}" + r=$? + set -e + if [ $r -eq 0 ]; then + debug "$i isn't targeted file" + continue + fi + NOT_MATCHED=("${NOT_MATCHED[@]/$n}") + fi + debug " $i" + res=`fc-validate -l $OPT_LANG $i || :` + if echo $res | grep -q Missing; then + echo "RESULT: FAIL: $i doesn't satisfy $OPT_LANG language coverage." + ret=1 + else + echo "RESULT: PASS: $i satisfy $OPT_LANG language coverage." + fi + fi + done +done + +exit $ret diff --git a/tests/roles/custom-test-fonts/meta/main.yml b/tests/roles/custom-test-fonts/meta/main.yml new file mode 100644 index 0000000..f6d273c --- /dev/null +++ b/tests/roles/custom-test-fonts/meta/main.yml @@ -0,0 +1,4 @@ +--- + +dependencies: + - role: str-common-init diff --git a/tests/roles/custom-test-fonts/tasks/main.yml b/tests/roles/custom-test-fonts/tasks/main.yml new file mode 100644 index 0000000..073fbf8 --- /dev/null +++ b/tests/roles/custom-test-fonts/tasks/main.yml @@ -0,0 +1,48 @@ +--- + +- block: + - name: language coverage + environment: + LANG: "en_US.UTF-8" + script: run-lang-coverage-test --lang "{{ item }}" {% if path_prefix is defined %} --path {{ path_prefix }} {% elif coverage.values is not defined or coverage[item].path_prefix is not defined %} {% else %} {{ '--path "' + (coverage[item].path_prefix | join('" --path "')) + '"' }} {% endif %} --artifactsdir "{{ remote_artifacts }}" {% if coverage.values is not defined or coverage[item].name is not defined %} {% else %} {{ "--name " + coverage[item].name }} {% endif %} {% if coverage.values is not defined or coverage[item].exclude is not defined %} {% else %} {{ '--exclude "' + (coverage[item].exclude | join('" --exclude "')) + '"' }} {% endif %} {% if coverage.values is not defined or coverage[item].include is not defined %} {% else %} {{ '--include "' + (coverage[item].include | join('" --include "')) + '"' }} {% endif %} {% if path_prefix is defined or coverage.values is defined and coverage[item].path_prefix is defined %} {% else %} {{ '--package "' + (required_packages|join('" --package "')) + '"' }} {% endif %} + with_items: + - "{{ coverage if coverage.keys is not defined else coverage.keys()|list }}" + - name: generic family assignment + environment: + LANG: "en_US.UTF-8" + when: families is defined + script: run-family-test --lang {{ item.lang }} --family '{{ item.family }}' --alias {{ item.alias }} --artifactsdir {{ remote_artifacts }} --package {{ package if item.package is not defined else item.package }} + with_items: + - "{{ families }}" + + - name: Check the results + shell: | + log="{{ remote_artifacts }}/test.log" + if [ ! -f "$log" ]; then + echo ERROR + echo "Test results not found." 1>&2 + elif grep ^ERROR "$log" 1>&2; then + echo ERROR + elif grep ^FAIL "$log" 1>&2; then + echo FAIL + elif grep -q ^PASS "$log"; then + echo PASS + else + echo ERROR + echo "No test results found." 1>&2 + fi + register: test_results + + - name: Set role result + set_fact: + role_result: "{{ test_results.stdout }}" + role_message: "{{ test_results.stderr|d('test execution error.') }}" + role_result_failed: "{{ test_results.stdout != 'PASS' }}" + role_result_msg: "{{ test_results.stderr|d('test execution error.') }}" + + - include_role: + name: str-common-final + + - name: Validate the result + shell: echo "test_results.stdout" + failed_when: test_results.stdout != 'PASS' diff --git a/tests/roles/fonts-testing-role/defaults/main.yml b/tests/roles/fonts-testing-role/defaults/main.yml deleted file mode 100644 index af4befe..0000000 --- a/tests/roles/fonts-testing-role/defaults/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -role_pkgs_req: - - rsync diff --git a/tests/roles/fonts-testing-role/meta/main.yml b/tests/roles/fonts-testing-role/meta/main.yml deleted file mode 100644 index 6626402..0000000 --- a/tests/roles/fonts-testing-role/meta/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -dependencies: - - role: str-common-init diff --git a/tests/roles/fonts-testing-role/tasks/main.yml b/tests/roles/fonts-testing-role/tasks/main.yml deleted file mode 100644 index 84bdbe0..0000000 --- a/tests/roles/fonts-testing-role/tasks/main.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -- name: Check if fontconfig is installed - register: fontconfig_testing_runner - find: - paths: "{{ ansible_env.PATH.split(':') }}" - pattern: fc-match - -- name: Install fontconfig - when: fontconfig_testing_runner.matched == 0 - block: - - name: Installing fontconfig - package: - name: - - fontconfig - -- name: Start fontconfig tests - block: - - name: Execute tests - shell: | - set -e - log_file="{{ remote_artifacts }}/{{ installed_test_name }}.log" - exec 2>>$log_file 1>>$log_file - echo "Starting fontconfig tests" - TEST=fontconfig - status="FAIL" - fc-validate -l th /usr/share/fonts/thai-scalable/Waree.ttf - if [ $? -eq 0 ]; then - status="PASS" - fi - echo "${status} $TEST" >> {{ remote_artifacts }}/test.log - - - name: Check the results - shell: grep "^FAIL" {{ remote_artifacts }}/test.log - register: test_fails - failed_when: False - - - name: Set role result - set_fact: - role_result_failed: "{{ (test_fails.stdout|d|length > 0) or (test_fails.stderr|d|length > 0) }}" - role_result_msg: "{{ test_fails.stdout|d('tests failed.') }}" - - - include_role: - name: str-common-final - diff --git a/tests/tests.yml b/tests/tests.yml index d075f20..41fce3f 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -1,11 +1,18 @@ - hosts: localhost + tags: + - classic roles: - - role: fonts-testing-role - installed_test_name: thai-scalable-fonts - tags: - - classic - - gating + - role: custom-test-fonts required_packages: - - fontconfig - - thai-scalable-waree-fonts - + - thai-scalable-waree-fonts + coverage: + th: + path_prefix: + - /usr/share/fonts/thai-scalable + include: + - Waree*.ttf + families: + - lang: th + package: thai-scalable-waree-fonts + alias: sans-serif + family: Waree