Add CI tests

This commit is contained in:
Peng Wu 2024-02-26 14:13:33 +08:00
parent 3540407a44
commit cf2f2e2cd3
7 changed files with 570 additions and 0 deletions

View File

@ -0,0 +1,67 @@
# Ansible role for tests using fontconfig
Put this role in your `tests.yml` playbook. The playbook will first install
package dependencies listed on playbook on your localhost, then it will proceed
to run testing. You can redefine the following variables:
* **artifacts**: An artifacts directory on localhost to store logs
* **remote_artifacts**: The directory on the system under test where the logs
are stored. Note: if this variable is left undefined, it will default to
`/tmp/artifacts`
* **required_packages**: A list of prerequisite packages required by tests.
Please note that for Atomic Host, additional packages will be installed
using the `rpm-ostree` command which is affecting the test subject (it's
similar as rebuilding an rpm package to be tested) so this should be used
with caution on only when necessary.
* **path_prefix**: The directory on the system where fonts are installed.
please use one in coverage sub-parameter if having different path_prefix
per sub-packages.
* **package**: The package name to test. this is used to find out fontconfig
config file. please use one in families sub-parameter if having different
config files per sub-packages.
* **coverage**: A list of languages for language coverage tests.
* **families**: A list of family test cases.
## Language coverage test parameters
Supporting two types of formats. one is a simple list of languages:
coverage:
- en
- fr
Another one is a dictionary that has a language as a key and values as parameters:
coverage:
en:
...
fr:
...
You can redefine the following variables for dictionary format:
* **exclude**: A list of font file names to exclude on this testing. this is
useful to avoid unexpected failures on iterating tests when a package has
multiple font files and has different coverages but you need to prevent
testing for few fonts which has different coverages to them.
Please note that the file name is relative to `path_prefix` parameter. also
good to consider using `include` if non-targeted files is more than targeted.
* **include**: A list of font file names to include on this testing. this is
useful to avoid unexpected failures on iterating tests when a pcakge has
multiple font files and has different coverages but you need to prevent
testing for few fonts which has different coverages to them.
Please note that the file name is relative to `path_prefix` parameter. also
good to consider using `exclude` if targeted files is more than non-targeted.
* **name**: The name to store logs. the test script is trying to make an unique
file names to store logs but not perfectly working in some cases. this is
optional parameter to make it unique by yourself.
* **path_prefix**: A list of directory names where fonts are installed on system.
## Family test parameters
* **lang**: A language to test family name for.
* **alias**: An alias name to test.
* **family**: A family name to test, which is supposed to be assinged to the alias.
* **package**: The package name to test. this is used to find out fontconfig
config file. this is optional. if not specified here, global `package`
parameter will be used.

View File

@ -0,0 +1,7 @@
---
role_pkgs_req:
- fontconfig
- fontconfig-devel
- pkg-config
- rsync

View File

@ -0,0 +1,162 @@
#! /bin/bash -efu
debug() {
if [ -n "$DEBUG" ]; then
echo "$*" >&2
fi
}
msg_usage() {
cat <<_EOF_
Run family test.
Usage:
$PROG <options>
Options:
-h, --help Display this help and exit
-v, --verbose Turn on debug
-l, --lang=LANG Test LANG language coverage (default: en)
-f, --family=FILE Set a family name supposed to be assigned for alias.
-g, --alias=STR Set an alias name. (default: sans-serif)
-a, --artifactsdir=DIR Set environment dir to store artifacts
-k, --package=NAME Set a package name for fonts.
_EOF_
}
PROG="${PROG:-${0##*/}}"
DEBUG="${DEBUG:-}"
OPT_LANG="${OPT_LANG:-en}"
OPT_FAMILY="${OPT_FAMILY:-}"
OPT_ARTIFACTS_DIR="${OPT_ARTIFACTS_DIR:-}"
OPT_ALIAS="${OPT_ALIAS:-sans-serif}"
OPT_PACKAGE="${OPT_PACKAGE:-}"
opt=$(getopt -n "$0" --options "hvl:f:t:a:g:k:" --longoptions "help,verbose,lang:,family:,test:,artifactsdir:,alias:,package:" -- "$@")
eval set -- "$opt"
while [[ $# -gt 0 ]]; do
case "$1" in
-k|--package)
OPT_PACKAGE="$2"
shift 2
;;
-g|--alias)
OPT_ALIAS="$2"
shift 2
;;
-a|--artifactsdir)
OPT_ARTIFACTS_DIR="$2"
shift 2
;;
-f|--family)
OPT_FAMILY="$2"
shift 2
;;
-l|--lang)
OPT_LANG="$2"
shift 2
;;
-v|--verbose)
DEBUG="-v"
shift
;;
-h|--help)
msg_usage
exit 0
;;
--)
shift
;;
*)
msg_usage
exit 1
esac
done
if [ -z "$OPT_ARTIFACTS_DIR" ] || [ -z "$OPT_LANG" ] || [ -z "$OPT_FAMILY" ]; then
echo "Use: $PROG -h for help."
exit 0
fi
debug "Alias: $OPT_ALIAS"
debug "Family: $OPT_FAMILY"
debug "Lang: $OPT_LANG"
debug "Artifacts dir: $OPT_ARTIFACTS_DIR"
debug "Package name: $OPT_PACKAGE"
STR_TEST_DASHED=$(echo "${OPT_PACKAGE}_${OPT_ALIAS}_${OPT_LANG}" | sed -e 's/\//-/g' -e 's/ /-/g')
debug "Log file: $STR_TEST_DASHED.log"
clean_exit() {
rc=$?;
trap - SIGINT SIGTERM SIGABRT EXIT
echo "Run test $OPT_ALIAS: done. Test's exit code: $rc"
for pid in $(ps -o pid --no-headers --ppid $$); do
if [ -n "$(ps -p $pid -o pid=)" ]; then
kill -s HUP $pid
fi
done
local log_file_name="$STR_TEST_DASHED.log"
local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name"
local status
if [[ $rc -eq 127 ]]; then
status="ERROR"
elif grep -q "RESULT: WARN" "$log_file_path"; then
status="ERROR"
elif grep -q "RESULT: FAIL" "$log_file_path"; then
status="FAIL"
elif grep -q "RESULT: PASS" "$log_file_path"; then
status="PASS"
elif grep -q "FAIL" "$log_file_path"; then
status="FAIL"
elif grep -q "PASS" "$log_file_path"; then
status="PASS"
else
status="ERROR"
fi
echo "$status $OPT_ALIAS" >> "$OPT_ARTIFACTS_DIR/test.log"
mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}"
local results="$OPT_ARTIFACTS_DIR/results.yml"
local result=$(echo $status | tr '[:upper:]' '[:lower:]')
test -f "$results" || echo 'results:' > "$results"
printf '%s\n' '' \
"- test: $OPT_ALIAS" \
" result: $result" \
" logs:" \
" - ${status}_${log_file_name}" \
>> "$results"
exit 0
}
trap clean_exit SIGINT SIGTERM SIGABRT EXIT
cachedir=`pkg-config --variable cachedir fontconfig`
tmpconfd=`mktemp --tmpdir -d fontsci.XXXXXXXX`
conf=$(for i in `rpm -ql $OPT_PACKAGE | grep conf.d`; do
echo "<include>$i</include>"
done)
cat <<_EOF_> $tmpconfd/fonts.conf
<fontconfig>
<dir>/usr/share/fonts</dir>
$conf
<cachedir>$cachedir</cachedir>
</fontconfig>
_EOF_
debug "Config: `cat $tmpconfd/fonts.conf`"
mkdir -p "$OPT_ARTIFACTS_DIR"
export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log"
logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log"
logfile="$(realpath "$logfile")"
exec > >(tee -a "$logfile") 2>&1
debug "Check family assignment"
res=`FONTCONFIG_FILE=$tmpconfd/fonts.conf fc-pattern -c -f "%{family}" :family=$OPT_ALIAS:lang=$OPT_LANG|grep "$OPT_FAMILY"`
ret=0
if [ "x$res" != "x" ]; then
echo "RESULT: PASS: $OPT_FAMILY was assigned to $OPT_ALIAS as expected"
else
echo "RESULT: FAIL: $OPT_FAMILY wasn't assigned to $OPT_ALIAS (actual result: $res)"
ret=1
fi
rm -rf $tmpconfd
exit $ret

View File

@ -0,0 +1,233 @@
#! /bin/bash -efu
debug() {
if [ -n "$DEBUG" ]; then
echo "$*" >&2
fi
}
msg_usage() {
cat <<_EOF_
Run language coverage test.
Usage:
$PROG <options>
Options:
-h, --help Display this help and exit
-v, --verbose Turn on debug
-l, --lang=LANG Test LANG language coverage (default: en)
-p, --path=PATH Test fonts on PATH
-n, --name=NAME Set NAME to store a log file.
-a, --artifactsdir=DIR test environment dir to store artifacts
-e, --exclude=FILE Exclude FILE to check.
-i, --include=FILE Include File to check.
_EOF_
}
PROG="${PROG:-${0##*/}}"
DEBUG="${DEBUG:-}"
OPT_LANG="${OPT_LANG:-en}"
OPT_PATH=()
OPT_ARTIFACTS_DIR="${OPT_ARTIFACTS_DIR:-}"
OPT_EXCLUDE=()
OPT_INCLUDE=()
OPT_NAME="${OPT_NAME:-}"
opt=$(getopt -n "$0" --options "hvl:p:n:a:e:i:" --longoptions "help,verbose,lang:,path:,name:,artifactsdir:,exclude:,include:" -- "$@")
eval set -- "$opt"
while [[ $# -gt 0 ]]; do
case "$1" in
-n|--name)
OPT_NAME="$2"
shift 2
;;
-i|--include)
OPT_INCLUDE+=("$2")
shift 2
;;
-e|--exclude)
OPT_EXCLUDE+=("$2")
shift 2
;;
-a|--artifactsdir)
OPT_ARTIFACTS_DIR="$2"
shift 2
;;
-p|--path)
OPT_PATH+=("$2")
shift 2
;;
-l|--lang)
OPT_LANG="$2"
shift 2
;;
-v|--verbose)
DEBUG="-v"
shift
;;
-h|--help)
msg_usage
exit 0
;;
--)
shift
;;
*)
msg_usage
exit 1
esac
done
if [ -z "$OPT_ARTIFACTS_DIR" ] || [ -z "$OPT_LANG" ] || [ -z "$OPT_PATH" ]; then
echo "Use: $PROG -h for help."
exit 0
fi
STR_TEST_DASHED=$(echo "${OPT_NAME:-$OPT_LANG}" | sed -e 's/\//-/g')
clean_exit() {
rc=$?;
trap - SIGINT SIGTERM SIGABRT EXIT
echo "Run test $OPT_LANG: done. Test's exit code: $rc"
for pid in $(ps -o pid --no-headers --ppid $$); do
if [ -n "$(ps -p $pid -o pid=)" ]; then
kill -s HUP $pid
fi
done
local log_file_name="$STR_TEST_DASHED.log"
local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name"
local status
if [[ $rc -eq 127 ]]; then
status="ERROR"
elif grep -q "RESULT: WARN" "$log_file_path"; then
status="ERROR"
elif grep -q "RESULT: FAIL" "$log_file_path"; then
status="FAIL"
elif grep -q "RESULT: PASS" "$log_file_path"; then
status="PASS"
elif grep -q "WARN" "$log_file_path"; then
status="ERROR"
elif grep -q "FAIL" "$log_file_path"; then
status="FAIL"
elif grep -q "PASS" "$log_file_path"; then
status="PASS"
else
status="ERROR"
fi
echo "$status $OPT_LANG" >> "$OPT_ARTIFACTS_DIR/test.log"
mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}"
local results="$OPT_ARTIFACTS_DIR/results.yml"
local result=$(echo $status | tr '[:upper:]' '[:lower:]')
test -f "$results" || echo 'results:' > "$results"
printf '%s\n' '' \
"- test: $OPT_LANG" \
" result: $result" \
" logs:" \
" - ${status}_${log_file_name}" \
>> "$results"
exit 0
}
trap clean_exit SIGINT SIGTERM SIGABRT EXIT
mkdir -p "$OPT_ARTIFACTS_DIR"
export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log"
logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log"
logfile="$(realpath "$logfile")"
exec > >(tee -a "$logfile") 2>&1
expand_regex() {
local p ret=()
local regex="$1"
shift
debug "Expanding $regex"
for p; do
set +f
debug "$p: $regex"
(cd $p;
local x=$(find -regextype posix-extended -regex "./$regex" -print|sed -e 's,^\./,,g')
debug "$x"
ret+=($x)
set -f
echo -n ${ret[@]}
)
done
echo -n ${ret[@]}
}
iv=()
ev=()
x=()
for p in ${OPT_INCLUDE[@]}; do
x=$(expand_regex $p ${OPT_PATH[@]})
if [ "x$x" == "x" ]; then
echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?"
continue
fi
iv=("${iv[@]}" "${x[@]}")
done
for p in ${OPT_EXCLUDE[@]}; do
x=$(expand_regex $p ${OPT_PATH[@]})
if [ "x$x" == "x" ]; then
echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?"
continue
fi
ev=("${ev[@]}" "${x[@]}")
done
OPT_EXCLUDE=(${ev[@]})
OPT_INCLUDE=(${iv[@]})
debug "Path: ${OPT_PATH[@]}"
debug "Lang: $OPT_LANG"
debug "Artifacts dir: $OPT_ARTIFACTS_DIR"
debug "Exclude: ${#OPT_EXCLUDE[@]}: ${OPT_EXCLUDE[@]}"
debug "Include: ${#OPT_INCLUDE[@]}: ${OPT_INCLUDE[@]}"
contains() {
local e match="$1"
shift
for e; do [[ "$e" == "$match" ]] && return 1; done
return 0
}
debug "Check language coverage"
ret=0
set +f
for p in ${OPT_PATH[@]}; do
for i in `find $p -regex '.*/*\.\(t1\)?\(ttf\)?\(otf\)?\(ttc\)?\(pcf.*\)?\(pfa\)?'`; do
set -f
debug "$i"
if test -f $i; then
n=`basename $i`
set +e
contains "$n" "${OPT_EXCLUDE[@]}"
ret=$?
set -e
if [ $ret -eq 1 ]; then
debug "ignoring $i"
continue
fi
if [ ${#OPT_INCLUDE[@]} -ne 0 ]; then
set +e
contains "$n" "${OPT_INCLUDE[@]}"
ret=$?
set -e
if [ $ret -eq 0 ]; then
debug "$i isn't targeted file"
continue
fi
NOT_MATCHED=("${NOT_MATCHED[@]/$n}")
fi
debug " $i"
res=`fc-validate -l $OPT_LANG $i || :`
if echo $res | grep -q Missing; then
echo "RESULT: FAIL: $i doesn't satisfy $OPT_LANG language coverage."
ret=1
else
echo "RESULT: PASS: $i satisfy $OPT_LANG language coverage."
fi
fi
done
done
exit $ret

View File

@ -0,0 +1,4 @@
---
dependencies:
- role: str-common-init

View File

@ -0,0 +1,52 @@
---
- block:
- name: language coverage
environment:
LANG: "en_US.UTF-8"
script: run-lang-coverage-test --lang "{{ item }}" {% if coverage.values is not defined or coverage[item].path_prefix is not defined %} --path {{ path_prefix }} {% else %} {{ '--path "' + (coverage[item].path_prefix | join('" --path "')) + '"' }} {% endif %} --artifactsdir "{{ remote_artifacts }}" {{ "" if coverage.values is not defined or coverage[item].name is not defined else "--name " + coverage[item].name }} {{ "" if coverage.values is not defined or coverage[item].exclude is not defined else '--exclude "' + (coverage[item].exclude | join('" --exclude "')) + '"'}} {{ "" if coverage.values is not defined or coverage[item].include is not defined else '--include "' + (coverage[item].include | join('" --include "')) + '"'}}
with_items:
- "{{ coverage if coverage.keys is not defined else coverage.keys()|list }}"
- name: generic family assignment
environment:
LANG: "en_US.UTF-8"
when: families is defined
script: run-family-test --lang {{ item.lang }} --family '{{ item.family }}' --alias {{ item.alias }} --artifactsdir {{ remote_artifacts }} --package {{ package if item.package is not defined else item.package }}
with_items:
- "{{ families }}"
- name: Collect all of failures into one file
shell: |
grep -v PASS "{{ remote_artifacts }}"/FAIL_*.log > "{{ remote_artifacts }}"/FAIL-all.log || :
- name: Check the results
shell: |
log="{{ remote_artifacts }}/test.log"
if [ ! -f "$log" ]; then
echo ERROR
echo "Test results not found." 1>&2
elif grep ^ERROR "$log" 1>&2; then
echo ERROR
elif grep ^FAIL "$log" 1>&2; then
echo FAIL
elif grep -q ^PASS "$log"; then
echo PASS
else
echo ERROR
echo "No test results found." 1>&2
fi
register: test_results
- name: Set role result
set_fact:
role_result: "{{ test_results.stdout }}"
role_message: "{{ test_results.stderr|d('test execution error.') }}"
role_result_failed: "{{ test_results.stdout != 'PASS' }}"
role_result_msg: "{{ test_results.stderr|d('test execution error.') }}"
- include_role:
name: str-common-final
- name: Validate the result
shell: echo "test_results.stdout"
failed_when: test_results.stdout != 'PASS'

45
tests/tests.yml Normal file
View File

@ -0,0 +1,45 @@
- hosts: localhost
tags:
- classic
roles:
- role: custom-test-fonts
required_packages:
- google-noto-serif-cjk-vf-fonts
coverage:
ja:
path_prefix:
- /usr/share/fonts/google-noto-serif-cjk-vf-fonts
include:
- NotoSerifCJK-VF.ttc
ko:
path_prefix:
- /usr/share/fonts/google-noto-serif-cjk-vf-fonts
include:
- NotoSerifCJK-VF.ttc
zh-cn:
path_prefix:
- /usr/share/fonts/google-noto-serif-cjk-vf-fonts
include:
- NotoSerifCJK-VF.ttc
zh-tw:
path_prefix:
- /usr/share/fonts/google-noto-serif-cjk-vf-fonts
include:
- NotoSerifCJK-VF.ttc
families:
- lang: ja
package: google-noto-serif-cjk-vf-fonts
alias: serif
family: Noto Serif CJK JP
- lang: ko
package: google-noto-serif-cjk-vf-fonts
alias: serif
family: Noto Serif CJK KR
- lang: zh-cn
package: google-noto-serif-cjk-vf-fonts
alias: serif
family: Noto Serif CJK SC
- lang: zh-tw
package: google-noto-serif-cjk-vf-fonts
alias: serif
family: Noto Serif CJK TC