Updates to L34 and L35.

Lectures/L34_BlackBoxTesting/BlackBoxTesting.tex

@@ -307,8 +307,9 @@ dist & PointT & real & ~\\
 \item Determining what the right answer should be is not always easy
 \begin{itemize}
 \item Air traffic control system
-\item Concurrent system
 \item Scientific computing
+\item Machine learning
+\item Artifical intelligence
 \end{itemize}
 \end{itemize}
 
@@ -332,6 +333,7 @@ dist & PointT & real & ~\\
 \begin{itemize}
 
 \item Using an independent program to approximate the oracle (pseudo oracle)
+\item Method of manufactured solutions
 \item Properties of the expected values can be easier than stating the expected
   output
 \bi
@@ -339,7 +341,7 @@ dist & PointT & real & ~\\
 \item \uncover<2->{List is sorted}
 \item \uncover<2->{Number of entries in file matches number of inputs}
 \item \uncover<2->{Conservation of energy or mass}
-\item \uncover<2->{Expected trends in output are observed}
+\item \uncover<2->{Expected trends in output are observed (metamorphic testing)}
 \item \uncover<2->{etc.}
 \ei
 \end{itemize}

Lectures/L35_Analysis/Analysis.tex

@@ -30,7 +30,7 @@
 
 \input{../def-beamer}
 
-\newcommand{\topic}{35 Analysis (Ch.\ 6) DRAFT}
+\newcommand{\topic}{35 Analysis (Ch.\ 6)}
 
 \input{../titlepage}
 
@@ -38,7 +38,8 @@
 
 \input{../footline}
 
-\lstset{language=java,breaklines=true,showspaces=false,showstringspaces=false,breakatwhitespace=true}
+\lstset{language=java, breaklines=true, showspaces=false,
+  showstringspaces=false, breakatwhitespace=true}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
@@ -60,7 +61,7 @@
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Administrative Details}
@@ -69,59 +70,41 @@
 
 \item Today's slide are partially based on slides by Dr.\ Wassyng
 
-\end{itemize}
-
-\end{frame}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\begin{frame}
-\frametitle{The Oracle Problem}
-
-\begin{itemize}
-
-\item Given input test cases that cover the domain, what are the expected outputs?
-\item Oracles are required at each stage of testing to tell us what the right answer is
-\item Black-box criteria are better than white-box for building test oracles
-\item Automated test oracles are required for running large amounts of tests
-\item Oracles are difficult to design - no universal recipe
-\end{itemize}
-
-\end{frame}
+\item No tutorials next week
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\item A4
+\bi
+\item Due April 3 at 11:59 pm
+\ei
 
-\begin{frame}
-\frametitle{The Oracle Problem Continued}
+\item Course evaluations
+\bi
+\item CS 2ME3: 41\%
+\item SE 2AA4: 54\%
+\item \href{https://evals.mcmaster.ca/login.php}{https://evals.mcmaster.ca/}
+\item Closes: 11:59 pm, Monday, April 10
+\ei
 
-\begin{itemize}
-
-\item Determining what the right answer should be is not always easy
-\begin{itemize}
-\item Air traffic control system
-\item Scientific computing
-\item Parallel testing can approximate an oracle
-\item Properties of the expected values can be easier than stating the expected output
-\end{itemize}
 \end{itemize}
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Module Testing}
 
 \begin{itemize}
 
-\item Scaffolding needed to create the environment in which the module should be tested
+\item Scaffolding needed to create the environment in which the module should be
+  tested
 \item Stubs - a module used by the module under test
 \item Driver - module activating the module under test
 \end{itemize}
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Testing a Functional Module}
@@ -130,7 +113,7 @@
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Integration Testing}
@@ -149,7 +132,7 @@
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Integration Testing and USES relation}
@@ -160,7 +143,22 @@
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Example}
+
+\begin{center}
+\includegraphics[scale=0.35]{../Figures/Example.png}
+\end{center}
+
+\begin{itemize}
+\item $M_1$ USES $M_2$ and $M_2$ IS\_COMPOSED\_OF \{$M_{2,1}$, $M_{2,2}$\}
+\item \structure{In what order would you test these modules?}
+\end{itemize}
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Example}
@@ -179,16 +177,17 @@
 \item Case 2
 \begin{itemize}
 \item Implement $M_{2,2}$ and test it by using a driver
-\item Implement $M_{2,1}$ and test the combination of $M_{2,1}$ and $M_{2,2}$ (i.e.\ $M_2$) by using a driver
+\item Implement $M_{2,1}$ and test the combination of $M_{2,1}$ and $M_{2,2}$
+  (i.e.\ $M_2$) by using a driver
 \item Finally implement $M_1$ and test it with $M_2$ using a driver for $M_1$
 \end{itemize}
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
-\frametitle{Testing OO Programs}
+\frametitle{Testing OO and Generic Programs}
 
 \begin{itemize}
 \item New issues
@@ -203,7 +202,7 @@
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Inheritance}
@@ -212,7 +211,7 @@
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{How to Test Classes of the Hierarchy}
@@ -222,38 +221,56 @@
 \end{center}
 
 \begin{itemize}
-\item ``Flattening'' the whole hierarchy and considering every class as totally independent component
+\item ``Flattening'' the whole hierarchy and considering every class as totally
+  independent component
 \item This does not exploit incrementality
 \item Finding an ad-hoc way to take advantage of the hierarchy
 \item Think about testing PointT.java and PointMassT.java
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{A Sample Strategy}
 
 \begin{itemize}
 \item A test that does not have to be repeated for any heir
-\item A test that must be performed for heir class X and all of its further heirs
-\item A test that must be redone by applying the same input data, but verifying that the output is not (or is) changed
-\item A test that must be modified by adding other input parameters and verifying that the the output changes accordingly
+\item A test that must be performed for heir class X and all of its further
+  heirs
+\item A test that must be redone by applying the same input data, but verifying
+  that the output is not (or is) changed
+\item A test that must be modified by adding other input parameters and
+  verifying that the the output changes accordingly
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Testing Concurrent and Real-time Systems}
 
 \begin{itemize}
 \item Nondeterminism inherent in concurrency affects repeatability
-\item For real-time systems, a test case consists not only of input data, but also of the times when such data are supplied
+\item For real-time systems, a test case consists not only of input data, but
+  also of the times when such data are supplied
 \item Considerable care and detail when testing real-time systems
 \end{itemize}
 \end{frame}
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Testing your Tests: Mutation Testing}
+\begin{itemize}
+\item Generate changes to the source code, called mutants, which become code faults
+\item Mutants include changing an operation, modifying constants, changing the order of execution, etc.
+\item The adequacy of a set of tests is established by running the tests on all generated mutants
+%\item Need to account for floating point approximations
+%\item See Hook and Kelly, 2009
+\end{itemize}
+\end{frame}
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
@@ -261,12 +278,13 @@
 
 \begin{itemize}
 \item Testing characterizes a \structure{single} execution
-\item Analysis characterizes a \structure{class} of executions; it is based on a \structure{model}
+\item Analysis characterizes a \structure{class} of executions; it is based on a
+  \structure{model}
 \item They have complementary advantages and disadvantages
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Informal Analysis Techniques and Code Walkthroughs}
@@ -275,17 +293,20 @@
 \item Recommended prescriptions
 \begin{itemize}
 \item Small number of people (three to five)
-\item Participants receive written documentation from the designer a few days before the meeting
+\item Participants receive written documentation from the designer a few days
+  before the meeting
 \item Predefined duration of the meeting (a few hours)
 \item Focus on the \structure{discovery} of errors, not on fixing them
 \item Participants: designer, moderator, and a secretary
 \item Foster cooperation; no evaluation of people
-\item Experience shows that most errors are discovered by the designer during the presentation, while trying to explain the design to other people
+\item Experience shows that most errors are discovered by the designer during
+  the presentation, while trying to explain the design to other people
 \end{itemize}
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
 \begin{frame}
 \frametitle{Informal Analysis Techniques Code Inspection}
 
@@ -301,33 +322,37 @@
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Correctness Proofs}
 
 \begin{itemize}
-\item Formal program analysis is a verification aid that may enhance program reliability
-\item Mathematically prove that the program's semantics implies its specification
+\item Formal program analysis is a verification aid that may enhance program
+  reliability
+\item Mathematically prove that the program's semantics implies its
+  specification
 \item Can use pre and post conditions
-\item Tabular expressions can be proven to match between specification of requirements and a specification of the design
+\item Tabular expressions can be proven to match between specification of
+  requirements and a specification of the design
 \item In many cases verification can be automated, at least partially
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Model Checking}
 
 \begin{itemize}
 \item Correctness verification, in general, is an undecidable problem
-\item Model checking is a rather recent verification technique based on the fact that most interesting system
-properties become decidable (algorithmically verifiable) when the system is modelled as a finite state machine
+\item Model checking is a rather recent verification technique based on the fact
+  that most interesting system properties become decidable (algorithmically
+  verifiable) when the system is modelled as a finite state machine
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Model Checking Continued}
@@ -342,13 +367,14 @@ properties become decidable (algorithmically verifiable) when the system is mode
 \item Verify whether the system's behaviour does indeed satisfy the desired property
 \begin{itemize}
 \item This step can be performed automatically
-\item The model checker either provides a \structure{proof} that the property holds or gives a \structure{counter
-example} in the form of a test case that exposes the system's failure to behave according to the property
+\item The model checker either provides a \structure{proof} that the property
+  holds or gives a \structure{counter example} in the form of a test case that
+  exposes the system's failure to behave according to the property
 \end{itemize}
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Why so Many Approaches to Testing and Analysis?}
@@ -366,7 +392,16 @@ View all of these as complementary
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Debugging}
+
+\structure{What approaches do you use for debugging?}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Debugging}
@@ -380,11 +415,14 @@ View all of these as complementary
 \item Intermediate assertions can help
 \item Tools like gdb, valgrind, etc.
 \end{itemize}
+\item Incremental integration tests helps
+\item Incrementally add complexity to test cases
+\item Like investigating an experiment - one controlled variable at a time
 \end{itemize}
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \end{document}