Skip to content

Commit 26afcea

Browse files
fs446fs446
committed
pca slides mods/improvements/bugfixes
1 parent 57f42f9 commit 26afcea

File tree

1 file changed

+242
-30
lines changed

1 file changed

+242
-30
lines changed

‎slides/ddasp_exercise_slides.tex

+242-30
Original file line numberDiff line numberDiff line change
@@ -1891,26 +1891,34 @@ \section{Section II: Feature Design}
18911891

18921892

18931893
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA)}
1894-
$\cdot$ for a $N \times F$ full-column rank matrix $\bm{X}$ we ensure that each column is mean-free by
1895-
$$\bm{X}_{N \times F} \leftarrow \bm{X}_{N \times F} - \frac{1}{N} \bm{1}_{1 \times N} \bm{X}_{N \times F}$$
18961894

1897-
$\cdot$ for a $F \times N$ full-row rank matrix $\bm{X}$ we ensure that each row is mean-free by
1898-
$$\bm{X}_{F \times N} \leftarrow \bm{X}_{F \times N} - \frac{1}{N} \bm{X}_{F \times N} \bm{1}_{N \times 1}$$
1895+
PCA is typically applied on mean-free data
18991896

1897+
$\cdot$ for an $N \times F$ full-column rank matrix $\bm{X}$ we ensure that each column is mean-free by
1898+
$$\bm{X}_{N \times F} \leftarrow \bm{X}_{N \times F} - \frac{1}{N} \bm{1}_{N \times N} \bm{X}_{N \times F}$$
1899+
1900+
$\cdot$ for an $F \times N$ full-row rank matrix $\bm{X}$ we ensure that each row is mean-free by
1901+
$$\bm{X}_{F \times N} \leftarrow \bm{X}_{F \times N} - \frac{1}{N} \bm{X}_{F \times N} \bm{1}_{N \times N}$$
1902+
1903+
PCA is often additionally performed on unit-variance preprocessed data, cf. function zscore()
1904+
1905+
this then yields a total variance of $$\mathrm{trace}(\mathrm{cov}(\mathrm{zscore}(\bm{X})))= F$$
1906+
1907+
which the PCA spreads over the principal component (PC) scores
19001908

19011909
\end{frame}
19021910

19031911

19041912

1905-
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) with SVD}
1913+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) via SVD}
19061914

19071915
$\cdot$ for $\bm{X}_c \in\mathbb{R}$, $\bm{X}_c = \bm{X}_r^\mathrm{T}$, $\bm{F}_c = \bm{F}_r^\mathrm{T}$, SVD matrices $\bm{U} \bm{\Sigma} \bm{V}^\mathrm{T}$ for $\bm{X}_c$
19081916

1909-
$\cdot$ PC Scores are ortho\underline{gonal} and variance-sorted, PC Loadings are ortho\underline{normal}
1917+
$\cdot$ PC scores are ortho\underline{gonal} and variance-sorted, PC loadings are ortho\underline{normal}
19101918

19111919
\vspace{0.5em}
19121920
\begin{minipage}[t]{0.49\textwidth}
1913-
full column rank, mean-free columns
1921+
full-column rank, mean-free columns
19141922
\begin{center}
19151923
$
19161924
\def\F{0.5}
@@ -1924,7 +1932,7 @@ \section{Section II: Feature Design}
19241932
\end{minipage}
19251933
%
19261934
\begin{minipage}[t]{0.49\textwidth}
1927-
full row rank, mean-free rows
1935+
full-row rank, mean-free rows
19281936
\begin{center}
19291937
$
19301938
\def\F{0.5}
@@ -1938,36 +1946,47 @@ \section{Section II: Feature Design}
19381946
\end{minipage}
19391947

19401948
\begin{minipage}[t]{0.49\textwidth}
1941-
$$\bm{X}_c= \bm{U} \bm{\Sigma} \bm{V}^\mathrm{T} = \bm{F}_c \bm{L}^\mathrm{T}$$
19421949

1943-
PC Scores $\bm{F}_c =
1950+
Mapping
1951+
1952+
$\bm{X}_c= \bm{U} \bm{\Sigma} \bm{V}^\mathrm{T} = \bm{F}_c \bm{L}^\mathrm{T}$
1953+
1954+
$\bm{F}_c = \bm{X}_c \bm{L} = \bm{X}_c \bm{V} = \bm{U} \bm{\Sigma}$
1955+
1956+
PC scores $\bm{F}_c =
19441957
\begin{bmatrix}
1945-
| & | & |\\
1946-
\bm{f}_1 & \bm{f}_: & \bm{f}_F\\
1947-
| & | & |\\
1958+
| & | & | & |\\
1959+
\bm{f}_1 & \bm{f}_2 & : & \bm{f}_F\\
1960+
| & | & | & |\\
19481961
\end{bmatrix}
19491962
= \bm{U} \bm{\Sigma}$
19501963

1951-
PC Loadings $$\bm{L} = \bm{V}$$
1964+
PC loadings $\bm{L} = \bm{V}$
1965+
19521966

1953-
Mapping $\bm{X}_c \bm{L} = \bm{X}_c \bm{V} = \bm{U} \bm{\Sigma} = \bm{F}_c$
19541967
\end{minipage}
19551968
%
19561969
\begin{minipage}[t]{0.49\textwidth}
1957-
$$\bm{X}_r = \bm{V} \bm{\Sigma} \bm{U}^\mathrm{T} = \bm{L} \bm{F}_r$$
19581970

1959-
PC Scores $\bm{F}_r =
1971+
Mapping
1972+
1973+
$\bm{X}_r = \bm{V} \bm{\Sigma} \bm{U}^\mathrm{T} = \bm{L} \bm{F}_r$
1974+
1975+
$\bm{F}_r = \bm{L}^\mathrm{T} \bm{X}_r = \bm{V}^\mathrm{T} \bm{X}_r = \bm{\Sigma} \bm{U}^\mathrm{T}$
1976+
1977+
PC scores $\bm{F}_r =
19601978
\begin{bmatrix}
19611979
- \bm{f}_1 -\\
1962-
- \bm{f}_: -\\
1980+
- \bm{f}_2 -\\
1981+
- : -\\
19631982
- \bm{f}_F -
19641983
\end{bmatrix}
19651984
=
19661985
\bm{\Sigma} \bm{U}^\mathrm{T}$
19671986

1968-
PC Loadings $$\bm{L} = \bm{V} $$
1987+
PC loadings $\bm{L} = \bm{V} $
1988+
19691989

1970-
Mapping $\bm{L}^\mathrm{T} \bm{X}_r = \bm{V}^\mathrm{T} \bm{X}_r = \bm{\Sigma} \bm{U}^\mathrm{T} = \bm{F}_r$
19711990

19721991
\end{minipage}
19731992

@@ -1983,11 +2002,11 @@ \section{Section II: Feature Design}
19832002

19842003
$\cdot$ for $\bm{X}_c \in\mathbb{R}$, $\bm{X}_c = \bm{X}_r^\mathrm{T}$, $\bm{F}_c = \bm{F}_r^\mathrm{T}$, SVD matrices $\bm{U} \bm{\Sigma} \bm{V}^\mathrm{T}$ for $\bm{X}_c$
19852004

1986-
$\cdot$ PC Scores are ortho\underline{gonal} and variance-sorted, PC Loadings are ortho\underline{normal}
2005+
$\cdot$ PC scores are ortho\underline{gonal} and variance-sorted, PC loadings are ortho\underline{normal}
19872006

19882007
\vspace{0.5em}
19892008
\begin{minipage}[t]{0.49\textwidth}
1990-
full column rank, mean-free columns
2009+
full-column rank, mean-free columns
19912010
\begin{center}
19922011
$
19932012
\def\F{0.5}
@@ -2001,7 +2020,7 @@ \section{Section II: Feature Design}
20012020
\end{minipage}
20022021
%
20032022
\begin{minipage}[t]{0.49\textwidth}
2004-
full row rank, mean-free rows
2023+
full-row rank, mean-free rows
20052024
\begin{center}
20062025
$
20072026
\def\F{0.5}
@@ -2021,9 +2040,9 @@ \section{Section II: Feature Design}
20212040

20222041
diagonalization (with SVD) $\bm{C}_X = \bm{V} \bm{\Lambda} \bm{V}^\mathrm{T}$
20232042

2024-
PC Scores $\bm{F}_c = \bm{X}_c \bm{V}$
2043+
PC scores $\bm{F}_c = \bm{X}_c \bm{V}$
20252044

2026-
PC Loadings $\bm{L} = \bm{V}$
2045+
PC loadings $\bm{L} = \bm{V}$
20272046

20282047
\end{minipage}
20292048
%
@@ -2032,26 +2051,219 @@ \section{Section II: Feature Design}
20322051

20332052
diagonalization (with SVD) $\bm{C}_X = \bm{V} \bm{\Lambda} \bm{V}^\mathrm{T}$
20342053

2035-
PC Scores $\bm{F}_r = \bm{V}^\mathrm{T} \bm{X}_r$
2054+
PC scores $\bm{F}_r = \bm{V}^\mathrm{T} \bm{X}_r$
20362055

2037-
PC Loadings $\bm{L} = \bm{V}$
2056+
PC loadings $\bm{L} = \bm{V}$
20382057

20392058
\end{minipage}
20402059

20412060
\vspace{0.5em}
20422061

20432062
%\small
2044-
$\cdot$ SVD-based diagonalization inherently sorts the eigenvalues in $\bm{\Lambda}$, making the PC Scores \underline{variance-sorted} (covariance matrix of $\bm{F}$ is a sorted diagonal matrix)
2063+
$\cdot$ an SVD-based diagonalization inherently sorts the eigenvalues in $\bm{\Lambda}$, making the orthogonal PC scores \underline{variance-sorted} (i.e. covariance matrix of $\bm{F}$ is a sorted diagonal matrix)
2064+
2065+
$\cdot$ $\bm{V} / \bm{F}$ might exhibit reflections compared to $\bm{V} / \bm{F}$ from SVD-based approach
2066+
2067+
$\cdot$ SVD / covariance approaches are consistent by itself as calculation of $\bm{F}$ and $\bm{L}$ is linked
2068+
2069+
\end{frame}
2070+
2071+
2072+
2073+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) Feature Representation}
2074+
2075+
$\cdot$ for $\bm{X}_c \in\mathbb{R}$, $\bm{X}_c = \bm{X}_r^\mathrm{T}$, $\bm{F}_c = \bm{F}_r^\mathrm{T}$, SVD matrices $\bm{U} \bm{\Sigma} \bm{V}^\mathrm{T}$ for $\bm{X}_c$
2076+
2077+
\begin{minipage}[t]{0.49\textwidth}
2078+
full-column rank, mean-free columns
2079+
2080+
$\bm{X}_{c,N \times F}$
2081+
2082+
PC scores $\bm{F}_c = \bm{X}_c \bm{V}$
2083+
2084+
PC loadings $\bm{L} = \bm{V}$
2085+
2086+
\end{minipage}
2087+
%
2088+
\begin{minipage}[t]{0.49\textwidth}
2089+
full-row rank, mean-free rows
2090+
2091+
$\bm{X}_{r,F \times N}$
2092+
2093+
PC scores $\bm{F}_r = \bm{V}^\mathrm{T} \bm{X}_r$
2094+
2095+
PC loadings $\bm{L} = \bm{V}$
2096+
2097+
\end{minipage}
2098+
2099+
\vspace{1em}
2100+
2101+
Reduced PC loading matrix
2102+
$\bm{V}_K =
2103+
\begin{bmatrix}
2104+
| & | & |\\
2105+
\bm{v}_1 & : & \bm{v}_{K \leq F}\\
2106+
| & | & |\\
2107+
\end{bmatrix}
2108+
$
2109+
allows for the following techniques
2110+
2111+
\vspace{1em}
2112+
2113+
$\cdot$ Low-Rank Approximation / Truncated SVD (yields a matrix with lower rank $K$)
2114+
2115+
\begin{minipage}[t]{0.49\textwidth}
2116+
2117+
$\tilde{\bm{X}}_{c,N \times F} = (\bm{X}_c \bm{V}_K) \bm{V}_K^\mathrm{T} = \sum_{i=1}^{K} \sigma_i \bm{u}_i \bm{v}_i^\mathrm{T}$
2118+
2119+
\end{minipage}
2120+
%
2121+
\begin{minipage}[t]{0.49\textwidth}
2122+
2123+
$\tilde{\bm{X}}_{r,F \times N} = \bm{V}_K (\bm{V}_K^\mathrm{T} \bm{X}_r) = \sum_{i=1}^{K} \sigma_i \bm{v}_i \bm{u}_i^\mathrm{T}$
2124+
2125+
\end{minipage}
2126+
2127+
\vspace{1em}
2128+
2129+
$\cdot$ Linear Dimensionality Reduction (yields a matrix with smaller dimension $K$)
2130+
2131+
\begin{minipage}[t]{0.49\textwidth}
2132+
$\tilde{\bm{X}}_{c,N \times K} = \bm{X}_c \bm{V}_K =
2133+
\begin{bmatrix}
2134+
| & | & |\\
2135+
\bm{f}_1 & : & \bm{f}_{K}\\
2136+
| & | & |\\
2137+
\end{bmatrix}
2138+
$
2139+
2140+
i.e. take only first $K$ columns of $\bm{F}_c$
2141+
\end{minipage}
2142+
%
2143+
\begin{minipage}[t]{0.49\textwidth}
2144+
$\tilde{\bm{X}}_{r,K \times N} = \bm{V}_K^\mathrm{T} \bm{X}_r
2145+
=
2146+
\begin{bmatrix}
2147+
- \bm{f}_1 -\\
2148+
- : -\\
2149+
- \bm{f}_K -
2150+
\end{bmatrix}
2151+
$
2152+
2153+
i.e. take only first $K$ rows of $\bm{F}_r$
2154+
\end{minipage}
2155+
2156+
\end{frame}
2157+
2158+
20452159

2046-
$\cdot$ vectors in $\bm{V}$ might exhibit reflections compared to $\bm{V}$ from SVD-based approach
20472160

2048-
$\cdot$ SVD / covariance approaches are consistent by itself as calculation of $\bm{F}$ and $\bm{L}$ are linked
20492161

2162+
2163+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) 2D-Data Example}
2164+
\begin{minipage}[t]{0.49\textwidth}
2165+
\includegraphics[width=\textwidth]{pca_2d_original_data.pdf}
2166+
\end{minipage}
2167+
%
2168+
\begin{minipage}[t]{0.49\textwidth}
2169+
\includegraphics[width=\textwidth]{pca_2d_original_data_with_pcdir.pdf}
2170+
\end{minipage}
2171+
\end{frame}
2172+
%%
2173+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) 2D-Data Example}
2174+
\begin{minipage}[t]{0.49\textwidth}
2175+
\includegraphics[width=\textwidth]{pca_2d_pc_data.pdf}
2176+
\end{minipage}
2177+
%
2178+
\begin{minipage}[t]{0.49\textwidth}
2179+
\includegraphics[width=\textwidth]{pca_2d_original_data_with_pcdir.pdf}
2180+
\end{minipage}
2181+
\end{frame}
2182+
%%
2183+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) 2D-Data Example}
2184+
\begin{minipage}[t]{0.49\textwidth}
2185+
\includegraphics[width=\textwidth]{pca_2d_truncated_svd.pdf}
2186+
\end{minipage}
2187+
%
2188+
\begin{minipage}[t]{0.49\textwidth}
2189+
\includegraphics[width=\textwidth]{pca_2d_original_data_with_pcdir.pdf}
2190+
\end{minipage}
20502191
\end{frame}
2192+
%%
2193+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) 2D-Data Example}
2194+
\begin{minipage}[t]{0.49\textwidth}
2195+
\includegraphics[width=\textwidth]{pca_2d_pc_data.pdf}
2196+
\end{minipage}
2197+
%
2198+
\begin{minipage}[t]{0.49\textwidth}
2199+
\includegraphics[width=\textwidth]{pca_2d_dim_red.pdf}
2200+
\end{minipage}
2201+
\end{frame}
2202+
2203+
2204+
20512205

20522206

20532207

20542208

2209+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) 3D-Data Example}
2210+
\begin{minipage}[t]{0.49\textwidth}
2211+
original data cloud in 3D space
2212+
2213+
\includegraphics[width=\textwidth]{pca_3d_original_data.pdf}
2214+
\end{minipage}
2215+
%
2216+
\begin{minipage}[t]{0.49\textwidth}
2217+
original data cloud in 3D space
2218+
2219+
\includegraphics[width=\textwidth]{pca_3d_original_data_with_pcdir.pdf}
2220+
\end{minipage}
2221+
\end{frame}
2222+
%%
2223+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) 3D-Data Example}
2224+
\begin{minipage}[t]{0.49\textwidth}
2225+
PC data cloud in 3D space
2226+
2227+
\includegraphics[width=\textwidth]{pca_3d_pc_data.pdf}
2228+
\end{minipage}
2229+
%
2230+
\begin{minipage}[t]{0.49\textwidth}
2231+
original data cloud in 3D space
2232+
2233+
\includegraphics[width=\textwidth]{pca_3d_original_data_with_pcdir.pdf}
2234+
\end{minipage}
2235+
\end{frame}
2236+
%%
2237+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) 3D-Data Example}
2238+
\begin{minipage}[t]{0.49\textwidth}
2239+
data \underline{plane} in 3D space
2240+
2241+
\includegraphics[width=\textwidth]{pca_3d_truncated_svd.pdf}
2242+
\end{minipage}
2243+
%
2244+
\begin{minipage}[t]{0.49\textwidth}
2245+
data cloud in 3D space
2246+
2247+
\includegraphics[width=\textwidth]{pca_3d_original_data_with_pcdir.pdf}
2248+
\end{minipage}
2249+
\end{frame}
2250+
%%
2251+
\begin{frame}[t]{Ex08: Principal Component Analysis (PCA) 3D-Data Example}
2252+
\begin{minipage}[t]{0.49\textwidth}
2253+
data cloud in 3D space
2254+
2255+
\includegraphics[width=\textwidth]{pca_3d_pc_data.pdf}
2256+
\end{minipage}
2257+
%
2258+
\begin{minipage}[t]{0.49\textwidth}
2259+
data \underline{plane} in 2D space (PC3 not used)
2260+
2261+
\includegraphics[width=\textwidth]{pca_3d_dim_red.pdf}
2262+
\end{minipage}
2263+
\end{frame}
2264+
2265+
2266+
20552267

20562268

20572269

0 commit comments

Comments
 (0)