aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMatt Fleming <matt.fleming@intel.com>2013-07-09 13:09:31 +0100
committerMatt Fleming <matt.fleming@intel.com>2013-07-23 14:58:41 +0100
commit21f8596c58a9791e7ea1f4ba93be17eed60a05b6 (patch)
tree267d3650d588a1855a48bd4423ebc99d27f45096
parentc6b80d76a6e1f76cdb5f1deaf298514843d0116a (diff)
downloadsyslinux-21f8596c58a9791e7ea1f4ba93be17eed60a05b6.tar.gz
syslinux-21f8596c58a9791e7ea1f4ba93be17eed60a05b6.tar.xz
syslinux-21f8596c58a9791e7ea1f4ba93be17eed60a05b6.zip
tests: Regression testsuite
A number of Syslinux releases have contained regressions when compared with previous versions. Now that the size of the derivative grid has exploded (PXELINUX, SYSLINUX,.... bios, efi32, efi64) we need to have some means of ensuring we don't introduce regressions so easily. Start building a suite of regression tests. Regression tests should test end-to-end functionality, e.g. loading a Linux kernel. If a test fails, that failure represents a regression in some behaviour, e.g. Syslinux behaves differently than it previously did. The idea is that when bugs are reported in Syslinux, we can create new tests that reproduce the buggy behaviour. Once a fix has been committed the new test should pass. From that point forward, the test provides a means of ensuring we never reintroduce that bug. The collection of tests illustrates the agreed upon behaviour of Syslinux. To execute the regression testsuite type, make regression Sample output might look like, Executing regression tests for SYSLINUX Running Linux kernel regression tests... [+] empty passed [+] kernelhello passed Running COM32 module regression tests... [+] chaindisk passed Executing regression tests for PXELINUX Running Linux kernel regression tests... [!] empty failed [+] kernelhello passed [+] pxetest passed Qemu is used to execute the regression tests in a virtual environment. Because it takes time to load Qemu, we can forgive the regression testsuite for not executing instantaneously. However, developers should keep in mind the idea of "failing fast" in their regression tests to minimize execution time - that is, if a test is going to fail, make it fail as quickly as possible to reduce the testrun time. It's possible to have derivative-specific tests, where the test only runs for a specific Syslinux boot loader. For an example see pxetest in tests/linux/Makefile. Currently the regression testsuite requires root privileges to execute and makes several calls to sudo(8). In future it would be nice to get rid of this requirement. Signed-off-by: Matt Fleming <matt.fleming@intel.com>
-rw-r--r--Makefile11
-rw-r--r--tests/Makefile79
-rw-r--r--tests/README15
-rwxr-xr-xtests/build-pxelinux51
-rwxr-xr-xtests/build-syslinux70
-rw-r--r--tests/com32/Makefile29
-rw-r--r--tests/com32/chaindisk.cfg7
-rw-r--r--tests/com32/chaindisk.results1
-rw-r--r--tests/linux/Makefile45
-rw-r--r--tests/linux/empty-vmlinuz0
-rw-r--r--tests/linux/empty.cfg3
-rw-r--r--tests/linux/empty.results2
-rw-r--r--tests/linux/kernelhello-vmlinuzbin0 -> 6843584 bytes
-rw-r--r--tests/linux/kernelhello.cfg5
-rw-r--r--tests/linux/kernelhello.results1
-rw-r--r--tests/linux/pxetest.cfg12
-rw-r--r--tests/linux/pxetest.results3
-rw-r--r--tests/recipes.mk71
-rw-r--r--tests/test.cfg6
19 files changed, 409 insertions, 2 deletions
diff --git a/Makefile b/Makefile
index a9b515c3..d90635af 100644
--- a/Makefile
+++ b/Makefile
@@ -72,7 +72,7 @@ include $(MAKEDIR)/syslinux.mk
-include $(OBJDIR)/version.mk
private-targets = prerel unprerel official release burn isolinux.iso \
- preupload upload
+ preupload upload test
ifeq ($(MAKECMDGOALS),)
MAKECMDGOALS += all
@@ -101,6 +101,13 @@ $(filter-out $(private-targets), $(MAKECMDGOALS)):
$(MAKE) -C $(OBJDIR) -f $(CURDIR)/Makefile SRC="$(topdir)" \
OBJ=$(OBJDIR) objdir=$(OBJDIR) $(MAKECMDGOALS)
+test:
+ printf "Executing unit tests\n"
+ $(MAKE) -C com32/lib/syslinux/tests all
+ $(MAKE) -C tests SRC="$(topdir)/tests" OBJ="$(topdir)/tests" \
+ objdir=$(OBJDIR) \
+ -f $(topdir)/tests/Makefile all
+
# Hook to add private Makefile targets for the maintainer.
-include $(topdir)/Makefile.private
@@ -206,7 +213,7 @@ NETINSTALLABLE = core/pxelinux.0 gpxe/gpxelinux.0 core/lpxelinux.0 \
endif # ifdef EFI_BUILD
-.PHONY: subdirs $(BSUBDIRS) $(ISUBDIRS)
+.PHONY: subdirs $(BSUBDIRS) $(ISUBDIRS) test
ifeq ($(HAVE_FIRMWARE),)
diff --git a/tests/Makefile b/tests/Makefile
new file mode 100644
index 00000000..99b1618f
--- /dev/null
+++ b/tests/Makefile
@@ -0,0 +1,79 @@
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+
+#
+# See linux/Makefile for an example Makefile. When tests are invoked the
+# location of the Syslinux files, as used by qemu, is in $(MOUNTDIR).
+# Tests that copy files into $(MOUNTDIR) must be sure to delete them once
+# the test has completed, otherwise the qemu harddisk image will become
+# full.
+#
+
+ifeq ($(INITIAL),)
+
+INITIAL := 1
+
+HDD = $(OBJ)/hdd.img
+
+export QEMU QEMU_FLAGS INITIAL HDD
+
+SYSLINUX_output-str := $(shell sudo $(SRC)/build-syslinux start \
+ $(objdir)/bios $(HDD) test.cfg)
+SYSLINUX_MOUNT_DIR = $(firstword $(SYSLINUX_output-str))
+SYSLINUX_INSTALL_DIR = $(SYSLINUX_MOUNT_DIR)/boot/syslinux
+SYSLINUX_CONFIG_FILE = $(SYSLINUX_INSTALL_DIR)/syslinux.cfg
+SYSLINUX_LOOPDEV =$(lastword $(SYSLINUX_output-str))
+
+PXELINUX_output-str := $(shell $(SRC)/build-pxelinux start $(topdir) test.cfg)
+PXELINUX_INSTALL_DIR = $(firstword $(PXELINUX_output-str))/tftpboot
+PXELINUX_CONFIG_FILE = $(PXELINUX_INSTALL_DIR)/pxelinux.cfg/default
+
+QEMU = qemu-system-i386
+
+all: syslinux-tests pxelinux-tests
+syslinux-tests:
+ $(MAKE) SRC="$(SRC)" OBJ="$(OBJ)" objdir="$(objdir)" \
+ INITIAL="$(INITIAL)" INSTALL_DIR="$(SYSLINUX_INSTALL_DIR)" \
+ CONFIG_FILE="$(SYSLINUX_CONFIG_FILE)" \
+ DERIVATIVE=SYSLINUX
+ @sudo $(SRC)/build-syslinux stop $(SYSLINUX_MOUNT_DIR) \
+ $(SYSLINUX_LOOPDEV) test.cfg
+
+pxelinux-tests:
+ $(MAKE) SRC="$(SRC)" OBJ="$(OBJ)" objdir="$(objdir)" \
+ INITIAL="$(INITIAL)" INSTALL_DIR="$(PXELINUX_INSTALL_DIR)" \
+ CONFIG_FILE="$(PXELINUX_CONFIG_FILE)" \
+ DERIVATIVE=PXELINUX
+ $(SRC)/build-pxelinux stop $(PXELINUX_INSTALL_DIR) test.cfg
+else # ifeq ($(INITIAL),)
+
+export DERIVATIVE
+ifeq ($(DERIVATIVE),SYSLINUX)
+QEMU_FLAGS += -hda $(HDD) -no-reboot
+else
+QEMU_FLAGS = -boot n -net nic,model=virtio -net user,tftp="$(INSTALL_DIR)",bootfile=pxelinux.0 -no-reboot
+endif
+
+all: banner subdirs
+
+banner:
+ printf "Executing regression tests for $(DERIVATIVE)\n"
+
+SUBDIRS = linux com32
+.PHONY: subdirs $(SUBDIRS) fini
+subdirs: $(SUBDIRS)
+$(SUBDIRS):
+ mkdir -p $(OBJ)/$@
+ $(MAKE) -C $(OBJ)/$@ SRC="$(SRC)"/$@ OBJ="$(OBJ)"/$@/ \
+ -f $(SRC)/$@/Makefile objdir="$(objdir)/tests" \
+ INSTALL_DIR="$(INSTALL_DIR)" CONFIG_FILE="$(CONFIG_FILE)" \
+ DERIVATIVE="$(DERIVATIVE)"
+
+endif # ifeq ($(INITIAL),)
diff --git a/tests/README b/tests/README
new file mode 100644
index 00000000..44575bcb
--- /dev/null
+++ b/tests/README
@@ -0,0 +1,15 @@
+Syslinux Regression Tests
+
+This directory contains acceptance or regression tests. Each test is
+intended to exercise a specific execution path through Syslinux, from
+user input to output. Unit tests are located in a module directory, e.g.
+com32/lib/syslinux/tests/, and test a specific "unit" of code such as a
+function or interface.
+
+The regression tests are intended to be run on the development
+environment. As such they must remain fast.
+
+Qemu must be installed and we grab OVMF from a predetermined location.
+
+If you're thinking of rewriting or refactoring a subsystem in a major
+way, please ensure there is a suitable test, and if not, write one.
diff --git a/tests/build-pxelinux b/tests/build-pxelinux
new file mode 100755
index 00000000..50c9de8c
--- /dev/null
+++ b/tests/build-pxelinux
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+
+usage()
+{
+ echo "Usage: " $0 "[start|stop] <srcdir> <config>" > /dev/stderr
+}
+
+if [ $# -lt 3 ]; then
+ usage
+ exit 1
+fi
+
+start_func()
+{
+ tmpdir=`mktemp -d || exit 1`
+ echo $tmpdir
+
+ mkdir -p $tmpdir/tftpboot/pxelinux.cfg
+ cp $2 $tmpdir/tftpboot/pxelinux.cfg/default
+
+ # This doesn't work if we built with O=/tmp/foobar
+ #make -p -s -C $1 bios netinstall INSTALLROOT=$tmpdir > /dev/stderr
+ for f in `find $1/bios -name "*.c32" -o -name "*.0"`; do
+ cp $f $tmpdir/tftpboot/
+ done
+}
+
+stop_func()
+{
+ rm -fr $1
+}
+
+case "$1" in
+ start)
+ shift
+ start_func $*
+ ;;
+ stop)
+ shift
+ stop_func $*
+ ;;
+esac
diff --git a/tests/build-syslinux b/tests/build-syslinux
new file mode 100755
index 00000000..a8972c0b
--- /dev/null
+++ b/tests/build-syslinux
@@ -0,0 +1,70 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+
+usage()
+{
+ echo "Usage: " $0 "[start|stop] <objdir> <outputfile> <config>" > /dev/stderr
+}
+
+if [ $# -lt 4 ]; then
+ usage
+ exit 1
+fi
+
+if [ `id -u` -ne 0 ]; then
+ echo $0 "must be invoked as root" > /dev/stderr
+ exit 1
+fi
+
+start_func()
+{
+ dd if=/dev/zero of=$2 bs=512 count=$(((32 * 1024 * 1024)/512)) > /dev/null 2>&1
+
+ tmpdir=`mktemp -d || exit 1`
+
+ next=`losetup -f`
+ losetup $next $2
+
+ mkfs.vfat $next > /dev/null 2>&1
+
+ mount $next $tmpdir
+
+ mkdir -p $tmpdir/boot/syslinux
+ cp $3 $tmpdir/boot/syslinux/syslinux.cfg
+
+ for f in `find $1 -name "*.c32"`; do
+ cp $f $tmpdir/boot/syslinux
+ done
+
+ $1/linux/syslinux --directory /boot/syslinux --install $next > /dev/stderr
+ sync
+
+ echo $tmpdir $next
+}
+
+stop_func()
+{
+ umount $1
+ rm -fr $1
+
+ losetup -d $2
+}
+
+case "$1" in
+ start)
+ shift
+ start_func $*
+ ;;
+ stop)
+ shift
+ stop_func $*
+ ;;
+esac
diff --git a/tests/com32/Makefile b/tests/com32/Makefile
new file mode 100644
index 00000000..5a9a95bd
--- /dev/null
+++ b/tests/com32/Makefile
@@ -0,0 +1,29 @@
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+
+include $(objdir)/recipes.mk
+
+all: tests
+
+ifeq ($(DERIVATIVE),SYSLINUX)
+chaindisk_cfg = chaindisk.cfg
+chaindisk_files = $(chaindisk_cfg)
+chaindisk_results = chaindisk.results
+chaindisk:
+ $(run-test)
+
+tests: banner chaindisk
+else
+
+tests:
+endif # ifeq ($(DERIVATIVE),SYSLINUX)
+
+banner:
+ printf " Running COM32 module regression tests...\n"
diff --git a/tests/com32/chaindisk.cfg b/tests/com32/chaindisk.cfg
new file mode 100644
index 00000000..037c3ec8
--- /dev/null
+++ b/tests/com32/chaindisk.cfg
@@ -0,0 +1,7 @@
+#
+# Chainload ourselves from the boot drive
+#
+DEFAULT -
+LABEL -
+ COM32 chain.c32
+ APPEND boot fs nohand break
diff --git a/tests/com32/chaindisk.results b/tests/com32/chaindisk.results
new file mode 100644
index 00000000..f081e7fc
--- /dev/null
+++ b/tests/com32/chaindisk.results
@@ -0,0 +1 @@
+Service-only run completed, exiting.
diff --git a/tests/linux/Makefile b/tests/linux/Makefile
new file mode 100644
index 00000000..bfa3ca75
--- /dev/null
+++ b/tests/linux/Makefile
@@ -0,0 +1,45 @@
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+# Regression tests for the Linux kernel loader code paths
+#
+
+include $(objdir)/recipes.mk
+
+ifeq ($(DERIVATIVE),PXELINUX)
+derivative-tests = pxetest
+
+pxetest_cfg = pxetest.cfg
+pxetest_files = $(pxetest_cfg) kernelhello-vmlinuz
+pxetest_results = pxetest.results
+
+endif
+
+all: tests
+
+empty_cfg = empty.cfg
+empty_files = $(empty_cfg) empty-vmlinuz
+empty_results = empty.results
+
+kernelhello_cfg = kernelhello.cfg
+kernelhello_files = $(kernelhello_cfg) kernelhello-vmlinuz
+kernelhello_results = kernelhello.results
+
+STANDARD_TESTS = kernelhello pxetest
+$(STANDARD_TESTS):
+ $(run-test)
+
+empty:
+ touch empty-vmlinuz
+ $(run-test)
+
+tests: banner empty kernelhello $(derivative-tests)
+
+banner:
+ printf " Running Linux kernel regression tests...\n"
diff --git a/tests/linux/empty-vmlinuz b/tests/linux/empty-vmlinuz
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/linux/empty-vmlinuz
diff --git a/tests/linux/empty.cfg b/tests/linux/empty.cfg
new file mode 100644
index 00000000..12cca7c2
--- /dev/null
+++ b/tests/linux/empty.cfg
@@ -0,0 +1,3 @@
+DEFAULT empty-linux
+LABEL empty-linux
+ KERNEL empty-vmlinuz
diff --git a/tests/linux/empty.results b/tests/linux/empty.results
new file mode 100644
index 00000000..fb512ec0
--- /dev/null
+++ b/tests/linux/empty.results
@@ -0,0 +1,2 @@
+
+Loading empty-vmlinuz... failed: I/O error
diff --git a/tests/linux/kernelhello-vmlinuz b/tests/linux/kernelhello-vmlinuz
new file mode 100644
index 00000000..68ae7ef8
--- /dev/null
+++ b/tests/linux/kernelhello-vmlinuz
Binary files differ
diff --git a/tests/linux/kernelhello.cfg b/tests/linux/kernelhello.cfg
new file mode 100644
index 00000000..3eb9e802
--- /dev/null
+++ b/tests/linux/kernelhello.cfg
@@ -0,0 +1,5 @@
+DEFAULT hello
+
+LABEL hello
+ KERNEL kernelhello-vmlinuz
+ APPEND console=ttyS0
diff --git a/tests/linux/kernelhello.results b/tests/linux/kernelhello.results
new file mode 100644
index 00000000..14c1f7d5
--- /dev/null
+++ b/tests/linux/kernelhello.results
@@ -0,0 +1 @@
+Hello, World!
diff --git a/tests/linux/pxetest.cfg b/tests/linux/pxetest.cfg
new file mode 100644
index 00000000..db954eb0
--- /dev/null
+++ b/tests/linux/pxetest.cfg
@@ -0,0 +1,12 @@
+DEFAULT dbg
+
+TIMEOUT 1
+ONTIMEOUT kernel
+
+LABEL dbg
+ COM32 debug.c32
+ APPEND -e pxelinux_scan_memory
+
+LABEL kernel
+ KERNEL kernelhello-vmlinuz
+ APPEND console=ttyS0 keeppxe
diff --git a/tests/linux/pxetest.results b/tests/linux/pxetest.results
new file mode 100644
index 00000000..f12bca6a
--- /dev/null
+++ b/tests/linux/pxetest.results
@@ -0,0 +1,3 @@
+Hello, World!
+Marking PXE code region 0x9c630 - 0x9cee8 as SMT_TERMINAL
+Marking PXE data region 0x9cef0 - 0x9fbf8 as SMT_TERMINAL
diff --git a/tests/recipes.mk b/tests/recipes.mk
new file mode 100644
index 00000000..f01c1add
--- /dev/null
+++ b/tests/recipes.mk
@@ -0,0 +1,71 @@
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+# Canned recipes
+
+#
+# copy-files - copy a config to the mounted filesystem
+#
+# Copies $(cfg) to the default Syslinux config location.
+# Usually this wants pairing with a use of $(remove-config).
+#
+# To use, your config filename must be the same as your target
+#
+define copy-files =
+ for f in $($@_files); do \
+ sudo cp $$f $(INSTALL_DIR) ;\
+ done
+ sync
+endef
+
+#
+# install-config
+#
+define install-config =
+ sudo sh -c 'echo INCLUDE $($@_cfg) >> $(CONFIG_FILE)'
+ sync
+endef
+
+#
+# remove-files - deletes files from the mounted filesystem
+#
+# Deletes $(mytest_files)
+#
+define remove-files =
+ for f in $($@_files); do \
+ sudo rm $(INSTALL_DIR)/$$f ;\
+ done
+endef
+
+#
+# delete-config - remove a test's config file from the master config
+#
+define delete-config =
+ sudo sed -i -e '/INCLUDE $($@_cfg)/d' $(CONFIG_FILE)
+endef
+
+#
+# run-test - begin executing the tests
+#
+define run-test =
+ $(copy-files)
+ $(install-config)
+
+ sudo $(QEMU) $(QEMU_FLAGS) -serial file:$@.log
+
+ $(delete-config)
+ $(remove-files)
+
+ sudo sort $@.log -o $@.log
+ if [ `comm -1 -3 $@.log $($@_results) | wc -l` -ne 0 ]; then \
+ printf " [!] $@ failed\n" ;\
+ else \
+ printf " [+] $@ passed\n" ;\
+ fi
+endef
diff --git a/tests/test.cfg b/tests/test.cfg
new file mode 100644
index 00000000..422603bb
--- /dev/null
+++ b/tests/test.cfg
@@ -0,0 +1,6 @@
+#
+# The default config file
+SERIAL 0
+ONERROR reboot.c32
+TIMEOUT 1
+ONTIMEOUT reboot.c32