Add CI tests

This commit is contained in:
Peng Wu 2019-12-10 16:53:05 +08:00
parent 937d97ef50
commit da2029cdff
10 changed files with 557 additions and 58 deletions

View File

@ -0,0 +1,69 @@
# Ansible role for tests using fontconfig
Put this role in your `tests.yml` playbook. The playbook will first install
package dependencies listed on playbook on your localhost, then it will proceed
to run testing. You can redefine the following variables:
* **artifacts**: An artifacts directory on localhost to store logs
* **remote_artifacts**: The directory on the system under test where the logs
are stored. Note: if this variable is left undefined, it will default to
`/tmp/artifacts`
* **required_packages**: A list of prerequisite packages required by tests.
Please note that for Atomic Host, additional packages will be installed
using the `rpm-ostree` command which is affecting the test subject (it's
similar as rebuilding an rpm package to be tested) so this should be used
with caution on only when necessary.
* **path_prefix**: The directory on the system where fonts are installed.
please use one in coverage sub-parameter if having different path_prefix
per sub-packages.
* **package**: The package name to test. this is used to find out fontconfig
config file. please use one in families sub-parameter if having different
config files per sub-packages.
* **coverage**: A list of languages for language coverage tests.
* **families**: A list of family test cases.
## Language coverage test parameters
Supporting two types of formats. one is a simple list of languages:
coverage:
- en
- fr
Another one is a dictionary that has a language as a key and values as parameters:
coverage:
en:
...
fr:
...
You can redefine the following variables for dictionary format:
* **exclude**: A list of font file names to exclude on this testing. this is
useful to avoid unexpected failures on iterating tests when a package has
multiple font files and has different coverages but you need to prevent
testing for few fonts which has different coverages to them.
Please note that the file name is relative to `path_prefix` parameter. also
good to consider using `include` if non-targeted files is more than targeted.
* **include**: A list of font file names to include on this testing. this is
useful to avoid unexpected failures on iterating tests when a pcakge has
multiple font files and has different coverages but you need to prevent
testing for few fonts which has different coverages to them.
Please note that the file name is relative to `path_prefix` parameter. also
good to consider using `exclude` if targeted files is more than non-targeted.
* **name**: The name to store logs. the test script is trying to make an unique
file names to store logs but not perfectly working in some cases. this is
optional parameter to make it unique by yourself.
* **path_prefix**: A list of directory names where fonts are installed on system.
this is optional parameter and tries to obtain the font paths from installed
packages by `required_packages` if not available.
## Family test parameters
* **lang**: A language to test family name for.
* **alias**: An alias name to test.
* **family**: A family name to test, which is supposed to be assinged to the alias.
* **package**: The package name to test. this is used to find out fontconfig
config file. this is optional. if not specified here, global `package`
parameter will be used.

View File

@ -0,0 +1,7 @@
---
role_pkgs_req:
- fontconfig
- fontconfig-devel
- pkg-config
- rsync

View File

@ -0,0 +1,162 @@
#! /bin/bash -efu
debug() {
if [ -n "$DEBUG" ]; then
echo "$*" >&2
fi
}
msg_usage() {
cat <<_EOF_
Run family test.
Usage:
$PROG <options>
Options:
-h, --help Display this help and exit
-v, --verbose Turn on debug
-l, --lang=LANG Test LANG language coverage (default: en)
-f, --family=FILE Set a family name supposed to be assigned for alias.
-g, --alias=STR Set an alias name. (default: sans-serif)
-a, --artifactsdir=DIR Set environment dir to store artifacts
-k, --package=NAME Set a package name for fonts.
_EOF_
}
PROG="${PROG:-${0##*/}}"
DEBUG="${DEBUG:-}"
OPT_LANG="${OPT_LANG:-en}"
OPT_FAMILY="${OPT_FAMILY:-}"
OPT_ARTIFACTS_DIR="${OPT_ARTIFACTS_DIR:-}"
OPT_ALIAS="${OPT_ALIAS:-sans-serif}"
OPT_PACKAGE="${OPT_PACKAGE:-}"
opt=$(getopt -n "$0" --options "hvl:f:t:a:g:k:" --longoptions "help,verbose,lang:,family:,test:,artifactsdir:,alias:,package:" -- "$@")
eval set -- "$opt"
while [[ $# -gt 0 ]]; do
case "$1" in
-k|--package)
OPT_PACKAGE="$2"
shift 2
;;
-g|--alias)
OPT_ALIAS="$2"
shift 2
;;
-a|--artifactsdir)
OPT_ARTIFACTS_DIR="$2"
shift 2
;;
-f|--family)
OPT_FAMILY="$2"
shift 2
;;
-l|--lang)
OPT_LANG="$2"
shift 2
;;
-v|--verbose)
DEBUG="-v"
shift
;;
-h|--help)
msg_usage
exit 0
;;
--)
shift
;;
*)
msg_usage
exit 1
esac
done
if [ -z "$OPT_ARTIFACTS_DIR" ] || [ -z "$OPT_LANG" ] || [ -z "$OPT_FAMILY" ]; then
echo "Use: $PROG -h for help."
exit 0
fi
debug "Alias: $OPT_ALIAS"
debug "Family: $OPT_FAMILY"
debug "Lang: $OPT_LANG"
debug "Artifacts dir: $OPT_ARTIFACTS_DIR"
debug "Package name: $OPT_PACKAGE"
STR_TEST_DASHED=$(echo "${OPT_PACKAGE}_${OPT_ALIAS}_${OPT_LANG}" | sed -e 's/\//-/g' -e 's/ /-/g')
debug "Log file: $STR_TEST_DASHED.log"
clean_exit() {
rc=$?;
trap - SIGINT SIGTERM SIGABRT EXIT
echo "Run test $OPT_ALIAS: done. Test's exit code: $rc"
for pid in $(ps -o pid --no-headers --ppid $$); do
if [ -n "$(ps -p $pid -o pid=)" ]; then
kill -s HUP $pid
fi
done
local log_file_name="$STR_TEST_DASHED.log"
local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name"
local status
if [[ $rc -eq 127 ]]; then
status="ERROR"
elif grep -q "RESULT: WARN" "$log_file_path"; then
status="ERROR"
elif grep -q "RESULT: FAIL" "$log_file_path"; then
status="FAIL"
elif grep -q "RESULT: PASS" "$log_file_path"; then
status="PASS"
elif grep -q "FAIL" "$log_file_path"; then
status="FAIL"
elif grep -q "PASS" "$log_file_path"; then
status="PASS"
else
status="ERROR"
fi
echo "$status $OPT_ALIAS" >> "$OPT_ARTIFACTS_DIR/test.log"
mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}"
local results="$OPT_ARTIFACTS_DIR/results.yml"
local result=$(echo $status | tr '[:upper:]' '[:lower:]')
test -f "$results" || echo 'results:' > "$results"
printf '%s\n' '' \
"- test: $OPT_ALIAS" \
" result: $result" \
" logs:" \
" - ${status}_${log_file_name}" \
>> "$results"
exit 0
}
trap clean_exit SIGINT SIGTERM SIGABRT EXIT
cachedir=`pkg-config --variable cachedir fontconfig`
tmpconfd=`mktemp --tmpdir -d fontsci.XXXXXXXX`
conf=$(for i in `rpm -ql $OPT_PACKAGE | grep conf.d`; do
echo "<include>$i</include>"
done)
cat <<_EOF_> $tmpconfd/fonts.conf
<fontconfig>
<dir>/usr/share/fonts</dir>
$conf
<cachedir>$cachedir</cachedir>
</fontconfig>
_EOF_
debug "Config: `cat $tmpconfd/fonts.conf`"
mkdir -p "$OPT_ARTIFACTS_DIR"
export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log"
logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log"
logfile="$(realpath "$logfile")"
exec > >(tee -a "$logfile") 2>&1
debug "Check family assignment"
res=`FONTCONFIG_FILE=$tmpconfd/fonts.conf fc-match -f "%{family[0]}" :family=$OPT_ALIAS:lang=$OPT_LANG`
ret=0
if [ "x$res" = "x$OPT_FAMILY" ]; then
echo "RESULT: PASS: $OPT_FAMILY was assigned to $OPT_ALIAS as expected"
else
echo "RESULT: FAIL: $OPT_FAMILY wasn't assigned to $OPT_ALIAS (actual result: $res)"
ret=1
fi
rm -rf $tmpconfd
exit $ret

View File

@ -0,0 +1,252 @@
#! /bin/bash -efu
debug() {
if [ -n "$DEBUG" ]; then
echo "$*" >&2
fi
}
msg_usage() {
cat <<_EOF_
Run language coverage test.
Usage:
$PROG <options>
Options:
-h, --help Display this help and exit
-v, --verbose Turn on debug
-l, --lang=LANG Test LANG language coverage (default: en)
-p, --path=PATH Test fonts on PATH
-k, --package=PACKAGE Specify PACKAGE to obtain some information
-n, --name=NAME Set NAME to store a log file.
-a, --artifactsdir=DIR test environment dir to store artifacts
-e, --exclude=FILE Exclude FILE to check.
-i, --include=FILE Include File to check.
_EOF_
}
PROG="${PROG:-${0##*/}}"
DEBUG="${DEBUG:-}"
OPT_LANG="${OPT_LANG:-en}"
OPT_PATH=()
OPT_PACKAGE=()
OPT_ARTIFACTS_DIR="${OPT_ARTIFACTS_DIR:-}"
OPT_EXCLUDE=()
OPT_INCLUDE=()
OPT_NAME="${OPT_NAME:-}"
opt=$(getopt -n "$0" --options "hvl:p:k:n:a:e:i:" --longoptions "help,verbose,lang:,path:,package:,name:,artifactsdir:,exclude:,include:" -- "$@")
eval set -- "$opt"
while [[ $# -gt 0 ]]; do
case "$1" in
-n|--name)
OPT_NAME="$2"
shift 2
;;
-i|--include)
OPT_INCLUDE+=("$2")
shift 2
;;
-e|--exclude)
OPT_EXCLUDE+=("$2")
shift 2
;;
-a|--artifactsdir)
OPT_ARTIFACTS_DIR="$2"
shift 2
;;
-p|--path)
OPT_PATH+=("$2")
shift 2
;;
-k|--package)
OPT_PACKAGE+=("$2")
shift 2
;;
-l|--lang)
OPT_LANG="$2"
shift 2
;;
-v|--verbose)
DEBUG="-v"
shift
;;
-h|--help)
msg_usage
exit 0
;;
--)
shift
;;
*)
msg_usage
exit 1
esac
done
if [ -z "$OPT_ARTIFACTS_DIR" ] || [ -z "$OPT_LANG" ] || [ ! -v OPT_PATH ] && [ ! -v OPT_PACKAGE ]; then
echo "Use: $PROG -h for help."
exit 0
fi
STR_TEST_DASHED=$(echo "${OPT_NAME:-$OPT_LANG}" | sed -e 's/\//-/g')
clean_exit() {
rc=$?;
trap - SIGINT SIGTERM SIGABRT EXIT
echo "Run test $OPT_LANG: done. Test's exit code: $rc"
for pid in $(ps -o pid --no-headers --ppid $$); do
if [ -n "$(ps -p $pid -o pid=)" ]; then
kill -s HUP $pid
fi
done
local log_file_name="$STR_TEST_DASHED.log"
local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name"
local status
if [[ $rc -eq 127 ]]; then
status="ERROR"
elif grep -q "RESULT: WARN" "$log_file_path"; then
status="ERROR"
elif grep -q "RESULT: FAIL" "$log_file_path"; then
status="FAIL"
elif grep -q "RESULT: PASS" "$log_file_path"; then
status="PASS"
elif grep -q "WARN" "$log_file_path"; then
status="ERROR"
elif grep -q "FAIL" "$log_file_path"; then
status="FAIL"
elif grep -q "PASS" "$log_file_path"; then
status="PASS"
else
status="ERROR"
fi
echo "$status $OPT_LANG" >> "$OPT_ARTIFACTS_DIR/test.log"
mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}"
local results="$OPT_ARTIFACTS_DIR/results.yml"
local result=$(echo $status | tr '[:upper:]' '[:lower:]')
test -f "$results" || echo 'results:' > "$results"
printf '%s\n' '' \
"- test: $OPT_LANG" \
" result: $result" \
" logs:" \
" - ${status}_${log_file_name}" \
>> "$results"
exit 0
}
trap clean_exit SIGINT SIGTERM SIGABRT EXIT
mkdir -p "$OPT_ARTIFACTS_DIR"
export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log"
logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log"
logfile="$(realpath "$logfile")"
exec > >(tee -a "$logfile") 2>&1
for pkg in ${OPT_PACKAGE[@]}; do
if ! rpm -q ${pkg} > /dev/null 2>&1; then
echo "Package isn't installed or maybe a typo: ${pkg}"
exit 127
fi
d=$(for d in $(rpm -ql ${pkg}|grep /usr/share/fonts); do
dirname $d
done | sort | uniq | grep /usr/share/fonts/)
if [[ ! " ${OPT_PATH[@]} " =~ " ${d} " ]]; then
OPT_PATH+=("$d")
fi
done
expand_regex() {
local p ret=()
local regex="$1"
shift
debug "Expanding $regex"
for p; do
set +f
debug "$p: $regex"
(cd $p;
local x=$(find -regextype posix-egrep -regex "./$regex" -print|sed -e 's,^\./,,g')
debug "$x"
ret+=($x)
set -f
echo -n ${ret[@]}
)
done
echo -n ${ret[@]}
}
iv=()
ev=()
x=()
for p in ${OPT_INCLUDE[@]}; do
x=$(expand_regex $p ${OPT_PATH[@]})
if [ "x$x" == "x" ]; then
echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?"
continue
fi
iv=("${iv[@]}" "${x[@]}")
done
for p in ${OPT_EXCLUDE[@]}; do
x=$(expand_regex $p ${OPT_PATH[@]})
if [ "x$x" == "x" ]; then
echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?"
continue
fi
ev=("${ev[@]}" "${x[@]}")
done
OPT_EXCLUDE=(${ev[@]})
OPT_INCLUDE=(${iv[@]})
debug "Path: ${OPT_PATH[@]}"
debug "Lang: $OPT_LANG"
debug "Artifacts dir: $OPT_ARTIFACTS_DIR"
debug "Exclude: ${#OPT_EXCLUDE[@]}: ${OPT_EXCLUDE[@]}"
debug "Include: ${#OPT_INCLUDE[@]}: ${OPT_INCLUDE[@]}"
contains() {
local e match="$1"
shift
for e; do [[ "$e" == "$match" ]] && return 1; done
return 0
}
debug "Check language coverage"
ret=0
set +f
for p in ${OPT_PATH[@]}; do
for i in `find $p -regex '.*/*\.\(t1\)?\(ttf\)?\(otf\)?\(ttc\)?\(pcf.*\)?\(pfa\)?'`; do
set -f
debug "$i"
if test -f $i; then
n=`basename $i`
set +e
contains "$n" "${OPT_EXCLUDE[@]}"
r=$?
set -e
if [ $r -eq 1 ]; then
debug "ignoring $i"
continue
fi
if [ ${#OPT_INCLUDE[@]} -ne 0 ]; then
set +e
contains "$n" "${OPT_INCLUDE[@]}"
r=$?
set -e
if [ $r -eq 0 ]; then
debug "$i isn't targeted file"
continue
fi
NOT_MATCHED=("${NOT_MATCHED[@]/$n}")
fi
debug " $i"
res=`fc-validate -l $OPT_LANG $i || :`
if echo $res | grep -q Missing; then
echo "RESULT: FAIL: $i doesn't satisfy $OPT_LANG language coverage."
ret=1
else
echo "RESULT: PASS: $i satisfy $OPT_LANG language coverage."
fi
fi
done
done
exit $ret

View File

@ -0,0 +1,4 @@
---
dependencies:
- role: str-common-init

View File

@ -0,0 +1,48 @@
---
- block:
- name: language coverage
environment:
LANG: "en_US.UTF-8"
script: run-lang-coverage-test --lang "{{ item }}" {% if path_prefix is defined %} --path {{ path_prefix }} {% elif coverage.values is not defined or coverage[item].path_prefix is not defined %} {% else %} {{ '--path "' + (coverage[item].path_prefix | join('" --path "')) + '"' }} {% endif %} --artifactsdir "{{ remote_artifacts }}" {% if coverage.values is not defined or coverage[item].name is not defined %} {% else %} {{ "--name " + coverage[item].name }} {% endif %} {% if coverage.values is not defined or coverage[item].exclude is not defined %} {% else %} {{ '--exclude "' + (coverage[item].exclude | join('" --exclude "')) + '"' }} {% endif %} {% if coverage.values is not defined or coverage[item].include is not defined %} {% else %} {{ '--include "' + (coverage[item].include | join('" --include "')) + '"' }} {% endif %} {% if path_prefix is defined or coverage.values is defined and coverage[item].path_prefix is defined %} {% else %} {{ '--package "' + (required_packages|join('" --package "')) + '"' }} {% endif %}
with_items:
- "{{ coverage if coverage.keys is not defined else coverage.keys()|list }}"
- name: generic family assignment
environment:
LANG: "en_US.UTF-8"
when: families is defined
script: run-family-test --lang {{ item.lang }} --family '{{ item.family }}' --alias {{ item.alias }} --artifactsdir {{ remote_artifacts }} --package {{ package if item.package is not defined else item.package }}
with_items:
- "{{ families }}"
- name: Check the results
shell: |
log="{{ remote_artifacts }}/test.log"
if [ ! -f "$log" ]; then
echo ERROR
echo "Test results not found." 1>&2
elif grep ^ERROR "$log" 1>&2; then
echo ERROR
elif grep ^FAIL "$log" 1>&2; then
echo FAIL
elif grep -q ^PASS "$log"; then
echo PASS
else
echo ERROR
echo "No test results found." 1>&2
fi
register: test_results
- name: Set role result
set_fact:
role_result: "{{ test_results.stdout }}"
role_message: "{{ test_results.stderr|d('test execution error.') }}"
role_result_failed: "{{ test_results.stdout != 'PASS' }}"
role_result_msg: "{{ test_results.stderr|d('test execution error.') }}"
- include_role:
name: str-common-final
- name: Validate the result
shell: echo "test_results.stdout"
failed_when: test_results.stdout != 'PASS'

View File

@ -1,2 +0,0 @@
role_pkgs_req:
- rsync

View File

@ -1,4 +0,0 @@
---
dependencies:
- role: str-common-init

View File

@ -1,44 +0,0 @@
---
- name: Check if fontconfig is installed
register: fontconfig_testing_runner
find:
paths: "{{ ansible_env.PATH.split(':') }}"
pattern: fc-validate
- name: Install fontconfig
when: fontconfig_testing_runner.matched == 0
block:
- name: Installing fontconfig
package:
name:
- fontconfig
- name: Start fontconfig tests
block:
- name: Execute tests
shell: |
set -e
log_file="{{ remote_artifacts }}/{{ installed_test_name }}.log"
exec 2>>$log_file 1>>$log_file
echo "Starting fontconfig tests"
TEST=fontconfig
status="FAIL"
fc-validate -l ii /usr/share/fonts/sil-nuosu/NuosuSIL.ttf
if [ $? -eq 0 ]; then
status="PASS"
fi
echo "${status} $TEST" >> {{ remote_artifacts }}/test.log
- name: Check the results
shell: grep "^FAIL" {{ remote_artifacts }}/test.log
register: test_fails
failed_when: False
- name: Set role result
set_fact:
role_result_failed: "{{ (test_fails.stdout|d|length > 0) or (test_fails.stderr|d|length > 0) }}"
role_result_msg: "{{ test_fails.stdout|d('tests failed.') }}"
- include_role:
name: str-common-final

View File

@ -1,11 +1,18 @@
- hosts: localhost
tags:
- classic
roles:
- role: fonts-testing-role
installed_test_name: sil-nuosu-fonts
tags:
- classic
- gating
- role: custom-test-fonts
required_packages:
- fontconfig
- sil-nuosu-fonts
- sil-nuosu-fonts
coverage:
ii:
path_prefix:
- /usr/share/fonts/sil-nuosu
include:
- NuosuSIL.ttf
families:
- lang: ii
package: sil-nuosu-fonts
alias: sans-serif
family: Nuosu SIL