Update to L32 - white box testing lecture.

Lectures/L32_WhiteBoxTesting/WhiteBoxTesting.tex

@@ -30,7 +30,7 @@
 
 \input{../def-beamer}
 
-\newcommand{\topic}{32 White Box Testing (Ch.\ 6) DRAFT}
+\newcommand{\topic}{32 White Box Testing (Ch.\ 6)}
 
 \input{../titlepage}
 
@@ -60,14 +60,102 @@
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Administrative Details}
 
+\begin{itemize}
+
+\item Problems with GitLab
+
+\item A4
+\bi
+\item \href{https://gitlab.cas.mcmaster.ca/Group7/Battleship}{Sample Design and
+    Documentation}
+\item \structure{Marking scheme has 175 potential marks (8/5)}
+\item \structure{Consider using a tool for code coverage metrics}
+\bi
+\item Coverage.py
+\item JCov
+\item etc.
+\ei
+\item Due April 3 at 11:59 pm
+\ei
+
+\item Course evaluations
+\bi
+\item \href{https://evals.mcmaster.ca/login.php}{https://evals.mcmaster.ca/}
+\item Opens: 10:00 am, Monday, March 27
+\item Closes: 11:59 pm, Monday, April 10
+\item \textbf{Your participation is highly valued!}
+\item Hi class participation means bonus marks
+\ei
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Continuous Integration Testing}
+\begin{itemize}
+\item \structure{What is continuous integration testing?}
+\end{itemize}
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Continuous Integration Testing}
+\begin{itemize}
+\item Information available on
+  \href{https://en.wikipedia.org/wiki/Continuous_integration}{Wikipedia}
+\item Developers integrate their code into a shared repo frequently (multiple
+  times a day)
+\item Each integration is automatically accompanied by regression tests and
+  other build tasks
+\item Build server
+\bi
+\item Unit tests
+\item Integration tests
+\item Static analysis
+\item Profile performance
+\item Extract documentation
+\item Update project web-page
+\item Portability tests
+\item etc.
+\ei
+\item Avoids potentially extreme problems with integration when the baseline and
+  a developer's code greatly differ
+\end{itemize}
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Continuous Integration Tools}
+\begin{itemize}
+\item Gitlab
+\bi
+\item Example at
+  \href{https://gitlab.cas.mcmaster.ca/andrem5/RogueReborn/pipelines}{Rogue
+    Reborn}
+\ei
+\item Jenkins
+\item Travis
+\item \href{https://www.docker.com/}{Docker}
+\bi
+\item Eliminates the ``it works on my machine'' problem
+\item Package dependencies with your apps
+\item A container for lightweight virtualization
+\item Not a full VM
+\ei
+\end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
 \begin{frame}
 \frametitle{Fault Testing}
 \begin{itemize}
@@ -77,11 +165,13 @@
 \item M = total number of fish caught in lake
 \item M' = number of marked fish caught
 \item T = (M - M')*N/M'
-\item Artificially seed faults, discover both seeded and new faults, estimate the total number of faults
+\item Artificially seed faults, discover both seeded and new faults, estimate
+  the total number of faults
+
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Fault Testing Continued}
@@ -98,7 +188,29 @@
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Fault Testing Usage Homework}
+After completion of a complex software project, two independent groups test
+it. The first group finds 300 errors and the second group finds 200 errors. A
+comparison of the errors discovered by the two teams shows that they found the
+same error in 50 cases. What is the range that estimates the number of errors in
+the original software project?
+
+% M1 = 300, M1'=50, N1 = 200, T1 = 1000
+% M2 = 200, M2'=50, N2 = 300, T2 = 900
+
+\be[A.]
+\item 900--1000 %answer
+\item -17--25
+\item 600--1500
+\item 0--150
+\item 150--250
+\ee
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Nonfunctional System Testing}
@@ -113,7 +225,7 @@ standards, procedures and guidelines
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Functional System Testing}
@@ -128,7 +240,7 @@ involves procedures, interfaces between people and the system, and training proc
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Functional System Testing Continued}
@@ -144,7 +256,7 @@ application or version of the application
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Testing Phases}
@@ -156,7 +268,7 @@ application or version of the application
 \end{enumerate}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Theoretical Foundations Of Testing: Definitions}
@@ -190,69 +302,86 @@ application or version of the application
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Definitions Questions}
+\begin{itemize}
+\item \structure{A test case t is an element of D or R?}
+\item \structure{A test set T is a finite subset of D or R?}
+\item \structure{How would we define whether a test is successful?}
+\item \structure{How would we define whether a test set is successful?}
+\end{itemize}
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Definitions Continued}
 \begin{itemize}
-\item Test case t
-\bi
-\item An element of D
-\ei
-\item Test set T
-\bi
-\item A finite subset of D
-\ei
+\item Test case t: An element of D
+\item Test set T: A finite subset of D
 \item Test is successful if P(t) is correct
 \item Test set successful if P correct for all t in T
 \end{itemize}
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Theoretical Foundations of Testing}
 
 \begin{itemize}
 \item Desire a test set $T$ that is a finite subset of $D$ that will uncover all errors
-\item Determining and ideal $T$ leads to several undecideable problems
-\item No algorithm exists to state if a test set will uncover all possible errors
-\item No algorithm exists to derive a test set that would prove program correctness
+\item Determining and ideal $T$ leads to several
+  \href{https://en.wikipedia.org/wiki/Undecidable_problem}{undecideable
+    problems}
+\item No algorithm exists:
+\bi
+\item To state if a test set will uncover all possible errors
+\item To derive a test set that would prove program correctness
+\item To determine whether suitable input exists to guarantee execution of a
+  given statement in a given program
+\item etc.
+\ei
 \end{itemize}
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Empirical Testing}
 
 \begin{itemize}
-\item Need to introduce empirical testing principles and heuristics as a compromise between the impossible and the
-inadequate
+\item Need to introduce empirical testing principles and heuristics as a
+  compromise between the impossible and the inadequate
 \item Find a strategy to select \structure{significant} test cases
-\item Significant means the test cases have a high potential of uncovering the presence of errors
+\item Significant means the test cases have a high potential of uncovering the
+  presence of errors
 \end{itemize}
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Complete-Coverage Principle}
 
 \begin{itemize}
-\item Try to group elements of $D$ into subdomains $D_1$, $D_2$, ..., $D_n$ where any element of each $D_i$ is likely
-to have similar behaviour
+\item Try to group elements of $D$ into subdomains $D_1$, $D_2$, ..., $D_n$
+  where any element of each $D_i$ is likely to have similar behaviour
 \item $D = D_1 \cup D_2 \cup ... \cup D_n$
 \item Select one test as a representative of the subdomain
-\item If $D_j \cap D_k = \emptyset$ for all $j \neq k$, (partition), any element can be chosen from each subdomain
-\item Otherwise choose representatives to minimize number of tests, yet fulfilling the principle
+\item If $D_j \cap D_k = \emptyset$ for all $j \neq k$, (partition), any element
+  can be chosen from each subdomain
+\item Otherwise choose representatives to minimize number of tests, yet
+  fulfilling the principle
 \end{itemize}
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Complete-Coverage Principle}
@@ -261,7 +390,19 @@ to have similar behaviour
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{White-box Testing}
+
+\begin{itemize}
+\item \structure{Intuitively, after running your test suites, what percentage of
+    the lines of code in your program should be exercised?}
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{White-box Coverage Testing}
@@ -279,7 +420,7 @@ to have similar behaviour
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Statement-Coverage Criterion}
@@ -292,7 +433,16 @@ coverage
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Example}
+
+\includegraphics[scale=0.5]{../Figures/ExampleStatementCoverageOnlyCode.png}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Example}
@@ -301,7 +451,16 @@ coverage
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\begin{frame}
+\frametitle{Weakness of the Criterion}
+
+\includegraphics[scale=0.6]{../Figures/WeaknessStatementCoverageOnlyCode.png}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Weakness of the Criterion}
@@ -310,7 +469,7 @@ coverage
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Edge-Coverage Criterion}
@@ -327,7 +486,7 @@ in $T$
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Control Graph Construction Rules}
@@ -336,7 +495,7 @@ in $T$
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Simplification}
@@ -348,7 +507,7 @@ A sequence of edges can be collapsed into just one edge\\
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Example: Euclid's Algorithm}
@@ -357,7 +516,7 @@ A sequence of edges can be collapsed into just one edge\\
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \begin{frame}
 \frametitle{Weakness}
@@ -367,7 +526,7 @@ A sequence of edges can be collapsed into just one edge\\
 
 \end{frame}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \end{document}