From 8b7c559f6c619bab8a708358db15b2959623c833 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Tue, 19 Feb 2019 14:54:31 +0100 Subject: [PATCH] v2v: add Var_expander This helper module provides a facility to replace %{FOO}-like variables in text strings with user-provided content. (cherry picked from commit a27748d7000f417c16045967497208d275a09ce8) --- .gitignore | 1 + v2v/Makefile.am | 32 ++++++++++- v2v/dummy.c | 2 + v2v/var_expander.ml | 72 ++++++++++++++++++++++++ v2v/var_expander.mli | 82 +++++++++++++++++++++++++++ v2v/var_expander_tests.ml | 113 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 v2v/dummy.c create mode 100644 v2v/var_expander.ml create mode 100644 v2v/var_expander.mli create mode 100644 v2v/var_expander_tests.ml diff --git a/.gitignore b/.gitignore index 637bf7765..f2efcdde2 100644 --- a/.gitignore +++ b/.gitignore @@ -693,6 +693,7 @@ Makefile.in /v2v/uefi.ml /v2v/uefi.mli /v2v/v2v_unit_tests +/v2v/var_expander_tests /v2v/virt-v2v /v2v/virt-v2v.1 /v2v/virt-v2v-copy-to-local diff --git a/v2v/Makefile.am b/v2v/Makefile.am index 2312812fb..f196be81d 100644 --- a/v2v/Makefile.am +++ b/v2v/Makefile.am @@ -98,6 +98,7 @@ SOURCES_MLI = \ utils.mli \ v2v.mli \ vCenter.mli \ + var_expander.mli \ windows.mli \ windows_virtio.mli @@ -106,6 +107,7 @@ SOURCES_ML = \ types.ml \ uefi.ml \ utils.ml \ + var_expander.ml \ python_script.ml \ name_from_disk.ml \ vCenter.ml \ @@ -442,7 +444,7 @@ TESTS += \ endif if HAVE_OCAML_PKG_OUNIT -TESTS += v2v_unit_tests +TESTS += v2v_unit_tests var_expander_tests endif if ENABLE_APPLIANCE @@ -651,7 +653,7 @@ EXTRA_DIST += \ # Unit tests. check_PROGRAMS = if HAVE_OCAML_PKG_OUNIT -check_PROGRAMS += v2v_unit_tests +check_PROGRAMS += v2v_unit_tests var_expander_tests endif v2v_unit_tests_BOBJECTS = \ @@ -671,13 +673,28 @@ v2v_unit_tests_SOURCES = $(virt_v2v_SOURCES) v2v_unit_tests_CPPFLAGS = $(virt_v2v_CPPFLAGS) v2v_unit_tests_CFLAGS = $(virt_v2v_CFLAGS) +var_expander_tests_BOBJECTS = \ + var_expander.cmo \ + var_expander_tests.cmo +var_expander_tests_XOBJECTS = $(var_expander_tests_BOBJECTS:.cmo=.cmx) + +var_expander_tests_SOURCES = dummy.c +var_expander_tests_CPPFLAGS = $(virt_v2v_CPPFLAGS) +var_expander_tests_CFLAGS = $(virt_v2v_CFLAGS) + if !HAVE_OCAMLOPT # Can't call this v2v_unit_tests_OBJECTS because automake gets confused. v2v_unit_tests_THEOBJECTS = $(v2v_unit_tests_BOBJECTS) v2v_unit_tests.cmo: OCAMLPACKAGES += -package oUnit + +var_expander_tests_THEOBJECTS = $(var_expander_tests_BOBJECTS) +var_expander_tests.cmo: OCAMLPACKAGES += -package oUnit else v2v_unit_tests_THEOBJECTS = $(v2v_unit_tests_XOBJECTS) v2v_unit_tests.cmx: OCAMLPACKAGES += -package oUnit + +var_expander_tests_THEOBJECTS = $(var_expander_tests_XOBJECTS) +var_expander_tests.cmx: OCAMLPACKAGES += -package oUnit endif v2v_unit_tests_DEPENDENCIES = \ @@ -696,6 +713,17 @@ v2v_unit_tests_LINK = \ $(OCAMLLINKFLAGS) \ $(v2v_unit_tests_THEOBJECTS) -o $@ +var_expander_tests_DEPENDENCIES = \ + $(var_expander_tests_THEOBJECTS) \ + ../common/mlpcre/mlpcre.$(MLARCHIVE) \ + $(top_srcdir)/ocaml-link.sh +var_expander_tests_LINK = \ + $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \ + $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) \ + $(OCAMLPACKAGES) -package oUnit \ + $(OCAMLLINKFLAGS) \ + $(var_expander_tests_THEOBJECTS) -o $@ + # Dependencies. .depend: \ $(srcdir)/*.mli \ diff --git a/v2v/dummy.c b/v2v/dummy.c new file mode 100644 index 000000000..ebab6198c --- /dev/null +++ b/v2v/dummy.c @@ -0,0 +1,2 @@ +/* Dummy source, to be used for OCaml-based tools with no C sources. */ +enum { foo = 1 }; diff --git a/v2v/var_expander.ml b/v2v/var_expander.ml new file mode 100644 index 000000000..24b9bafe3 --- /dev/null +++ b/v2v/var_expander.ml @@ -0,0 +1,72 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Std_utils + +exception Invalid_variable of string + +let var_re = PCRE.compile "(^|[^%])%{([^}]+)}" + +let check_variable var = + String.iter ( + function + | '0'..'9' + | 'a'..'z' + | 'A'..'Z' + | '_' + | '-' -> () + | _ -> raise (Invalid_variable var) + ) var + +let scan_variables str = + let res = ref [] in + let offset = ref 0 in + while PCRE.matches ~offset:!offset var_re str; do + let var = PCRE.sub 2 in + check_variable var; + let _, end_ = PCRE.subi 0 in + List.push_back res var; + offset := end_ + done; + List.remove_duplicates !res + +let replace_fn str fn = + let res = ref str in + let offset = ref 0 in + while PCRE.matches ~offset:!offset var_re !res; do + let var = PCRE.sub 2 in + check_variable var; + let start_, end_ = PCRE.subi 0 in + match fn var with + | None -> + offset := end_ + | Some text -> + let prefix_len = + let prefix_start, prefix_end = PCRE.subi 1 in + prefix_end - prefix_start in + res := (String.sub !res 0 (start_ + prefix_len)) ^ text ^ (String.sub !res end_ (String.length !res - end_)); + offset := start_ + prefix_len + String.length text + done; + !res + +let replace_list str lst = + let fn var = + try Some (List.assoc var lst) + with Not_found -> None + in + replace_fn str fn diff --git a/v2v/var_expander.mli b/v2v/var_expander.mli new file mode 100644 index 000000000..80aa33c2c --- /dev/null +++ b/v2v/var_expander.mli @@ -0,0 +1,82 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** Simple variable expander. + + This module provides the support to expand variables in strings, + specified in the form of [%{name}]. + + For example: + +{v +let str = "variable-%{INDEX} in %{INDEX} replaced %{INDEX} times" +let index = ref 0 +let fn = function + | "INDEX" -> + incr index; + Some (string_of_int !index) + | _ -> None +in +let str = Var_expander.replace_fn str fn +(* now str is "variable-1 in 2 replaced 3 times" *) +v} + + The names of variables can contain only ASCII letters (uppercase, + and lowercase), digits, underscores, and dashes. + + The replacement is done in a single pass: this means that if a + variable is replaced with the text of a variable, that new text + is kept as is in the final output. In practice: + +{v +let str = "%{VAR}" +let str = Var_expander.replace_list str [("VAR", "%{VAR}")] +(* now str is "%{VAR}" *) +v} +*) + +exception Invalid_variable of string +(** Invalid variable name error. + + In case a variable contains characters not allowed, then this + exception with the actual unacceptable variable. *) + +val scan_variables : string -> string list +(** Scan the pattern string for all the variables available. + + This can raise {!Invalid_variable} in case there are invalid + variable names. *) + +val replace_fn : string -> (string -> string option) -> string +(** Replaces a string expanding all the variables. + + The replacement function specify how a variable is replaced; + if [None] is returned, then that variable is not replaced. + + This can raise {!Invalid_variable} in case there are invalid + variable names. *) + +val replace_list : string -> (string * string) list -> string +(** Replaces a string expanding all the variables. + + The replacement list specify how a variable is replaced; + if it is not specified in the list, then that variable is not + replaced. + + This can raise {!Invalid_variable} in case there are invalid + variable names. *) diff --git a/v2v/var_expander_tests.ml b/v2v/var_expander_tests.ml new file mode 100644 index 000000000..35b628369 --- /dev/null +++ b/v2v/var_expander_tests.ml @@ -0,0 +1,113 @@ +(* virt-v2v + * Copyright (C) 2019 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Printf +open OUnit + +open Std_utils + +let assert_equal_string = assert_equal ~printer:identity +let assert_equal_stringlist = assert_equal ~printer:(fun x -> "(" ^ (String.escaped (String.concat "," x)) ^ ")") + +let replace_none_fn _ = None +let replace_empty_fn _ = Some "" + +let test_no_replacement () = + assert_equal_string "" (Var_expander.replace_fn "" replace_none_fn); + assert_equal_string "x" (Var_expander.replace_fn "x" replace_none_fn); + assert_equal_string "%{}" (Var_expander.replace_fn "%{}" replace_none_fn); + assert_equal_string "%{EMPTY}" (Var_expander.replace_fn "%{EMPTY}" replace_none_fn); + assert_equal_string "%{EMPTY} %{no}" (Var_expander.replace_fn "%{EMPTY} %{no}" replace_none_fn); + assert_equal_string "a %{EMPTY} b" (Var_expander.replace_fn "a %{EMPTY} b" replace_none_fn); + () + +let test_replacements () = + assert_equal_string "" (Var_expander.replace_fn "%{EMPTY}" replace_empty_fn); + assert_equal_string "x " (Var_expander.replace_fn "x %{EMPTY}" replace_empty_fn); + assert_equal_string "xy" (Var_expander.replace_fn "x%{EMPTY}y" replace_empty_fn); + assert_equal_string "x<->y" (Var_expander.replace_fn "x%{FOO}y" (function | "FOO" -> Some "<->" | _ -> None)); + assert_equal_string "a x b" (Var_expander.replace_fn "a %{FOO} b" (function | "FOO" -> Some "x" | _ -> None)); + assert_equal_string "%{FOO} x" (Var_expander.replace_fn "%{FOO} %{BAR}" (function | "BAR" -> Some "x" | _ -> None)); + assert_equal_string "%{FOO}" (Var_expander.replace_fn "%{BAR}" (function | "BAR" -> Some "%{FOO}" | _ -> None)); + assert_equal_string "%{FOO} x" (Var_expander.replace_fn "%{BAR} %{FOO}" (function | "BAR" -> Some "%{FOO}" | "FOO" -> Some "x" | _ -> None)); + begin + let str = "%{INDEX}, %{INDEX}, %{INDEX}" in + let index = ref 0 in + let fn = function + | "INDEX" -> + incr index; + Some (string_of_int !index) + | _ -> None + in + assert_equal_string "1, 2, 3" (Var_expander.replace_fn str fn) + end; + () + +let test_escape () = + assert_equal_string "%%{FOO}" (Var_expander.replace_fn "%%{FOO}" replace_empty_fn); + assert_equal_string "x %%{FOO} x" (Var_expander.replace_fn "%{FOO} %%{FOO} %{FOO}" (function | "FOO" -> Some "x" | _ -> None)); + () + +let test_list () = + assert_equal_string "x %{NONE}" (Var_expander.replace_list "%{FOO} %{NONE}" [("FOO", "x")]); + () + +let test_scan_variables () = + let assert_invalid_variable var = + let str = "%{" ^ var ^ "}" in + assert_raises (Var_expander.Invalid_variable var) + (fun () -> Var_expander.scan_variables str) + in + assert_equal_stringlist [] (Var_expander.scan_variables ""); + assert_equal_stringlist [] (Var_expander.scan_variables "foo"); + assert_equal_stringlist ["FOO"] (Var_expander.scan_variables "%{FOO}"); + assert_equal_stringlist ["FOO"; "BAR"] (Var_expander.scan_variables "%{FOO} %{BAR}"); + assert_equal_stringlist ["FOO"; "BAR"] (Var_expander.scan_variables "%{FOO} %{BAR} %{FOO}"); + assert_equal_stringlist ["FOO"; "BAR"] (Var_expander.scan_variables "%{FOO} %%{ESCAPED} %{BAR}"); + assert_invalid_variable "FOO/BAR"; + () + +let test_errors () = + let assert_invalid_variable var = + let str = "%{" ^ var ^ "}" in + assert_raises (Var_expander.Invalid_variable var) + (fun () -> Var_expander.replace_fn str replace_none_fn) + in + assert_invalid_variable "FOO/BAR"; + assert_invalid_variable "FOO:BAR"; + assert_invalid_variable "FOO(BAR"; + assert_invalid_variable "FOO)BAR"; + assert_invalid_variable "FOO@BAR"; + () + +(* Suites declaration. *) +let suite = + TestList ([ + "basic" >::: [ + "no_replacement" >:: test_no_replacement; + "replacements" >:: test_replacements; + "escape" >:: test_escape; + "list" >:: test_list; + "scan_variables" >:: test_scan_variables; + "errors" >:: test_errors; + ]; + ]) + +let () = + ignore (run_test_tt_main suite); + Printf.fprintf stderr "\n" -- 2.18.4