diff --git a/Lectures/L34_BlackBoxTesting/BlackBoxTesting.pdf b/Lectures/L34_BlackBoxTesting/BlackBoxTesting.pdf
index fe629546ae4a24068f6b7af8193c87685bc0406e..a2f044c9f8ad3efbc7db8fc3d4214407aa928e3d 100644
Binary files a/Lectures/L34_BlackBoxTesting/BlackBoxTesting.pdf and b/Lectures/L34_BlackBoxTesting/BlackBoxTesting.pdf differ
diff --git a/Lectures/L34_BlackBoxTesting/BlackBoxTesting.tex b/Lectures/L34_BlackBoxTesting/BlackBoxTesting.tex
index 2befc539c0e5af4a2ce81cf9f7584b7e84716328..af0867819907a829dcd50b3f6bdf4f38bf17b26a 100755
--- a/Lectures/L34_BlackBoxTesting/BlackBoxTesting.tex
+++ b/Lectures/L34_BlackBoxTesting/BlackBoxTesting.tex
@@ -307,8 +307,9 @@ dist & PointT & real & ~\\
\item Determining what the right answer should be is not always easy
\begin{itemize}
\item Air traffic control system
-\item Concurrent system
\item Scientific computing
+\item Machine learning
+\item Artifical intelligence
\end{itemize}
\end{itemize}
@@ -332,6 +333,7 @@ dist & PointT & real & ~\\
\begin{itemize}
\item Using an independent program to approximate the oracle (pseudo oracle)
+\item Method of manufactured solutions
\item Properties of the expected values can be easier than stating the expected
output
\bi
@@ -339,7 +341,7 @@ dist & PointT & real & ~\\
\item \uncover<2->{List is sorted}
\item \uncover<2->{Number of entries in file matches number of inputs}
\item \uncover<2->{Conservation of energy or mass}
-\item \uncover<2->{Expected trends in output are observed}
+\item \uncover<2->{Expected trends in output are observed (metamorphic testing)}
\item \uncover<2->{etc.}
\ei
\end{itemize}
diff --git a/Lectures/L35_Analysis/Analysis.pdf b/Lectures/L35_Analysis/Analysis.pdf
index 3cda1182d723d51b38fecaca179da1b44d6833fc..2d61a8b2f778ee5c49a4a555cbe7f45c90feb3ef 100644
Binary files a/Lectures/L35_Analysis/Analysis.pdf and b/Lectures/L35_Analysis/Analysis.pdf differ
diff --git a/Lectures/L35_Analysis/Analysis.tex b/Lectures/L35_Analysis/Analysis.tex
index cd8da138be1f5fb95564bd8321862f1db7bb5484..9bb6a101d7ed54b0c1dc2f0b10832fa9e081e502 100755
--- a/Lectures/L35_Analysis/Analysis.tex
+++ b/Lectures/L35_Analysis/Analysis.tex
@@ -30,7 +30,7 @@
\input{../def-beamer}
-\newcommand{\topic}{35 Analysis (Ch.\ 6) DRAFT}
+\newcommand{\topic}{35 Analysis (Ch.\ 6)}
\input{../titlepage}
@@ -38,7 +38,8 @@
\input{../footline}
-\lstset{language=java,breaklines=true,showspaces=false,showstringspaces=false,breakatwhitespace=true}
+\lstset{language=java, breaklines=true, showspaces=false,
+ showstringspaces=false, breakatwhitespace=true}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -60,7 +61,7 @@
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Administrative Details}
@@ -69,59 +70,41 @@
\item Today's slide are partially based on slides by Dr.\ Wassyng
-\end{itemize}
-
-\end{frame}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\begin{frame}
-\frametitle{The Oracle Problem}
-
-\begin{itemize}
-
-\item Given input test cases that cover the domain, what are the expected outputs?
-\item Oracles are required at each stage of testing to tell us what the right answer is
-\item Black-box criteria are better than white-box for building test oracles
-\item Automated test oracles are required for running large amounts of tests
-\item Oracles are difficult to design - no universal recipe
-\end{itemize}
-
-\end{frame}
+\item No tutorials next week
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\item A4
+\bi
+\item Due April 3 at 11:59 pm
+\ei
-\begin{frame}
-\frametitle{The Oracle Problem Continued}
+\item Course evaluations
+\bi
+\item CS 2ME3: 41\%
+\item SE 2AA4: 54\%
+\item \href{https://evals.mcmaster.ca/login.php}{https://evals.mcmaster.ca/}
+\item Closes: 11:59 pm, Monday, April 10
+\ei
-\begin{itemize}
-
-\item Determining what the right answer should be is not always easy
-\begin{itemize}
-\item Air traffic control system
-\item Scientific computing
-\item Parallel testing can approximate an oracle
-\item Properties of the expected values can be easier than stating the expected output
-\end{itemize}
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Module Testing}
\begin{itemize}
-\item Scaffolding needed to create the environment in which the module should be tested
+\item Scaffolding needed to create the environment in which the module should be
+ tested
\item Stubs - a module used by the module under test
\item Driver - module activating the module under test
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Testing a Functional Module}
@@ -130,7 +113,7 @@
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Integration Testing}
@@ -149,7 +132,7 @@
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Integration Testing and USES relation}
@@ -160,7 +143,22 @@
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Example}
+
+\begin{center}
+\includegraphics[scale=0.35]{../Figures/Example.png}
+\end{center}
+
+\begin{itemize}
+\item $M_1$ USES $M_2$ and $M_2$ IS\_COMPOSED\_OF \{$M_{2,1}$, $M_{2,2}$\}
+\item \structure{In what order would you test these modules?}
+\end{itemize}
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Example}
@@ -179,16 +177,17 @@
\item Case 2
\begin{itemize}
\item Implement $M_{2,2}$ and test it by using a driver
-\item Implement $M_{2,1}$ and test the combination of $M_{2,1}$ and $M_{2,2}$ (i.e.\ $M_2$) by using a driver
+\item Implement $M_{2,1}$ and test the combination of $M_{2,1}$ and $M_{2,2}$
+ (i.e.\ $M_2$) by using a driver
\item Finally implement $M_1$ and test it with $M_2$ using a driver for $M_1$
\end{itemize}
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
-\frametitle{Testing OO Programs}
+\frametitle{Testing OO and Generic Programs}
\begin{itemize}
\item New issues
@@ -203,7 +202,7 @@
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Inheritance}
@@ -212,7 +211,7 @@
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{How to Test Classes of the Hierarchy}
@@ -222,38 +221,56 @@
\end{center}
\begin{itemize}
-\item ``Flattening'' the whole hierarchy and considering every class as totally independent component
+\item ``Flattening'' the whole hierarchy and considering every class as totally
+ independent component
\item This does not exploit incrementality
\item Finding an ad-hoc way to take advantage of the hierarchy
\item Think about testing PointT.java and PointMassT.java
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{A Sample Strategy}
\begin{itemize}
\item A test that does not have to be repeated for any heir
-\item A test that must be performed for heir class X and all of its further heirs
-\item A test that must be redone by applying the same input data, but verifying that the output is not (or is) changed
-\item A test that must be modified by adding other input parameters and verifying that the the output changes accordingly
+\item A test that must be performed for heir class X and all of its further
+ heirs
+\item A test that must be redone by applying the same input data, but verifying
+ that the output is not (or is) changed
+\item A test that must be modified by adding other input parameters and
+ verifying that the the output changes accordingly
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Testing Concurrent and Real-time Systems}
\begin{itemize}
\item Nondeterminism inherent in concurrency affects repeatability
-\item For real-time systems, a test case consists not only of input data, but also of the times when such data are supplied
+\item For real-time systems, a test case consists not only of input data, but
+ also of the times when such data are supplied
\item Considerable care and detail when testing real-time systems
\end{itemize}
\end{frame}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Testing your Tests: Mutation Testing}
+\begin{itemize}
+\item Generate changes to the source code, called mutants, which become code faults
+\item Mutants include changing an operation, modifying constants, changing the order of execution, etc.
+\item The adequacy of a set of tests is established by running the tests on all generated mutants
+%\item Need to account for floating point approximations
+%\item See Hook and Kelly, 2009
+\end{itemize}
+\end{frame}
+
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
@@ -261,12 +278,13 @@
\begin{itemize}
\item Testing characterizes a \structure{single} execution
-\item Analysis characterizes a \structure{class} of executions; it is based on a \structure{model}
+\item Analysis characterizes a \structure{class} of executions; it is based on a
+ \structure{model}
\item They have complementary advantages and disadvantages
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Informal Analysis Techniques and Code Walkthroughs}
@@ -275,17 +293,20 @@
\item Recommended prescriptions
\begin{itemize}
\item Small number of people (three to five)
-\item Participants receive written documentation from the designer a few days before the meeting
+\item Participants receive written documentation from the designer a few days
+ before the meeting
\item Predefined duration of the meeting (a few hours)
\item Focus on the \structure{discovery} of errors, not on fixing them
\item Participants: designer, moderator, and a secretary
\item Foster cooperation; no evaluation of people
-\item Experience shows that most errors are discovered by the designer during the presentation, while trying to explain the design to other people
+\item Experience shows that most errors are discovered by the designer during
+ the presentation, while trying to explain the design to other people
\end{itemize}
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
\begin{frame}
\frametitle{Informal Analysis Techniques Code Inspection}
@@ -301,33 +322,37 @@
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Correctness Proofs}
\begin{itemize}
-\item Formal program analysis is a verification aid that may enhance program reliability
-\item Mathematically prove that the program's semantics implies its specification
+\item Formal program analysis is a verification aid that may enhance program
+ reliability
+\item Mathematically prove that the program's semantics implies its
+ specification
\item Can use pre and post conditions
-\item Tabular expressions can be proven to match between specification of requirements and a specification of the design
+\item Tabular expressions can be proven to match between specification of
+ requirements and a specification of the design
\item In many cases verification can be automated, at least partially
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Model Checking}
\begin{itemize}
\item Correctness verification, in general, is an undecidable problem
-\item Model checking is a rather recent verification technique based on the fact that most interesting system
-properties become decidable (algorithmically verifiable) when the system is modelled as a finite state machine
+\item Model checking is a rather recent verification technique based on the fact
+ that most interesting system properties become decidable (algorithmically
+ verifiable) when the system is modelled as a finite state machine
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Model Checking Continued}
@@ -342,13 +367,14 @@ properties become decidable (algorithmically verifiable) when the system is mode
\item Verify whether the system's behaviour does indeed satisfy the desired property
\begin{itemize}
\item This step can be performed automatically
-\item The model checker either provides a \structure{proof} that the property holds or gives a \structure{counter
-example} in the form of a test case that exposes the system's failure to behave according to the property
+\item The model checker either provides a \structure{proof} that the property
+ holds or gives a \structure{counter example} in the form of a test case that
+ exposes the system's failure to behave according to the property
\end{itemize}
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Why so Many Approaches to Testing and Analysis?}
@@ -366,7 +392,16 @@ View all of these as complementary
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Debugging}
+
+\structure{What approaches do you use for debugging?}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Debugging}
@@ -380,11 +415,14 @@ View all of these as complementary
\item Intermediate assertions can help
\item Tools like gdb, valgrind, etc.
\end{itemize}
+\item Incremental integration tests helps
+\item Incrementally add complexity to test cases
+\item Like investigating an experiment - one controlled variable at a time
\end{itemize}
\end{frame}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\end{document}