Chapter 6: The determinant¶
1. Linear Systems as Transformations¶
The Big Idea¶
A system of linear equations like:
Can be written as a matrix equation:
Or more compactly: \(Ax = v\)
Geometric interpretation: We’re looking for a vector \(x\) which, after applying transformation \(A\), lands on \(v\).
Think of it like this:
\(A\) is a transformation
\(v\) is our target destination
\(x\) is the input that gets us there
We’re asking: “What lands on \(v\)?”
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import FancyArrowPatch
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
sns.set_style("whitegrid")
plt.rcParams['figure.figsize'] = (10, 8)
def draw_vector(ax, origin, vector, color='blue', label=None, width=0.01):
"""Draw a vector as an arrow."""
arrow = FancyArrowPatch(
origin, origin + vector,
mutation_scale=20, lw=2, arrowstyle='-|>', color=color, label=label
)
ax.add_patch(arrow)
def setup_ax(ax, xlim=(-5, 5), ylim=(-5, 5)):
"""Setup 2D axis."""
ax.set_xlim(xlim)
ax.set_ylim(ylim)
ax.set_aspect('equal')
ax.axhline(y=0, color='k', linewidth=0.5)
ax.axvline(x=0, color='k', linewidth=0.5)
ax.grid(True, alpha=0.3)
ax.set_xlabel('x', fontsize=12)
ax.set_ylabel('y', fontsize=12)
def visualize_linear_system():
"""Visualize solving Ax = v as a geometric problem."""
# Define the system
A = np.array([[2, 5], [3, -2]])
v = np.array([-3, 8])
# Solve for x
x = np.linalg.solve(A, v)
fig, axes = plt.subplots(1, 3, figsize=(15, 5))
# Left: The unknown x
ax1 = axes[0]
setup_ax(ax1)
draw_vector(ax1, np.array([0, 0]), x, color='green', label=f'x = [{x[0]:.2f}, {x[1]:.2f}]')
ax1.set_title('1. Unknown vector x', fontsize=14, fontweight='bold')
ax1.legend(fontsize=11)
# Middle: Apply transformation A
ax2 = axes[1]
setup_ax(ax2, xlim=(-10, 10), ylim=(-10, 10))
# Draw transformed basis vectors
i_transformed = A @ np.array([1, 0])
j_transformed = A @ np.array([0, 1])
draw_vector(ax2, np.array([0, 0]), i_transformed, color='red', label="A's i-hat")
draw_vector(ax2, np.array([0, 0]), j_transformed, color='blue', label="A's j-hat")
ax2.set_title('2. Transformation A', fontsize=14, fontweight='bold')
ax2.legend(fontsize=11)
# Right: Result Ax = v
ax3 = axes[2]
setup_ax(ax3, xlim=(-10, 10), ylim=(-10, 10))
Ax = A @ x
draw_vector(ax3, np.array([0, 0]), Ax, color='purple', label=f'Ax = v = [{Ax[0]:.2f}, {Ax[1]:.2f}]')
draw_vector(ax3, np.array([0, 0]), v, color='orange', label='Target v', width=0.015)
ax3.set_title('3. Result: x lands on v!', fontsize=14, fontweight='bold')
ax3.legend(fontsize=11)
plt.tight_layout()
plt.show()
print(f"System: Ax = v")
print(f"A = \n{A}")
print(f"\nTarget v = {v}")
print(f"\nSolution x = {x}")
print(f"\nVerification: Ax = {A @ x}")
print(f"Match: {np.allclose(A @ x, v)}")
visualize_linear_system()
2. Matrix Inverses: Rewinding Transformations¶
The Inverse Transformation¶
If determinant is non-zero, we can “rewind” the transformation:
\(A^{-1}\) is the inverse matrix - it undoes what \(A\) does.
Key property: \(A^{-1}A = I\) (the identity matrix)
The identity matrix \(I\) represents “doing nothing”:
When Does an Inverse Exist?¶
det(A) ≠ 0: Inverse exists, unique solution
det(A) = 0: No inverse, space is squished
def visualize_inverse():
"""Visualize a transformation and its inverse."""
# A 90-degree rotation
theta = np.pi / 2
A = np.array([[np.cos(theta), -np.sin(theta)],
[np.sin(theta), np.cos(theta)]])
A_inv = np.linalg.inv(A)
# Test vector
v = np.array([3, 1])
v_transformed = A @ v
v_back = A_inv @ v_transformed
fig, axes = plt.subplots(1, 3, figsize=(15, 5))
# Original
ax1 = axes[0]
setup_ax(ax1)
draw_vector(ax1, np.array([0, 0]), v, color='green', label='Original v')
ax1.set_title('1. Start with v', fontsize=14, fontweight='bold')
ax1.legend(fontsize=11)
# After A
ax2 = axes[1]
setup_ax(ax2)
draw_vector(ax2, np.array([0, 0]), v_transformed, color='blue', label='After A (rotated 90°)')
ax2.set_title('2. Apply A (rotate)', fontsize=14, fontweight='bold')
ax2.legend(fontsize=11)
# After A^(-1)
ax3 = axes[2]
setup_ax(ax3)
draw_vector(ax3, np.array([0, 0]), v_back, color='green', label='After A⁻¹ (back to original!)')
ax3.set_title('3. Apply A⁻¹ (undo rotation)', fontsize=14, fontweight='bold')
ax3.legend(fontsize=11)
plt.tight_layout()
plt.show()
print(f"A (rotation 90°):")
print(f"{A}\n")
print(f"A⁻¹ (rotation -90°):")
print(f"{A_inv}\n")
print(f"A⁻¹ × A = I (identity):")
print(f"{A_inv @ A}\n")
print(f"Verification: v → A(v) → A⁻¹(A(v)) = {v_back}")
print(f"Matches original: {np.allclose(v, v_back)}")
visualize_inverse()
3. When Determinant is Zero: No Inverse¶
The Problem¶
When det(A) = 0, the transformation squishes space into a lower dimension:
2D → line
3D → plane or line
You cannot unsquish a line back into a plane!
Solutions in Zero-Determinant Cases¶
Even with det(A) = 0, solutions might exist if you’re lucky:
If \(v\) happens to lie in the squished space, solution exists
If \(v\) is outside the squished space, no solution
def visualize_zero_determinant():
"""Show what happens when determinant is zero."""
# Matrix that squishes to a line
A = np.array([[2, 4], [1, 2]]) # Second column is 2x first
print(f"Matrix A:")
print(f"{A}")
print(f"\nDeterminant: {np.linalg.det(A):.6f}")
print(f"This is zero! Space will be squished.\n")
# Create a grid of vectors
x = np.linspace(-2, 2, 20)
y = np.linspace(-2, 2, 20)
X, Y = np.meshgrid(x, y)
# Transform the grid
points = np.array([X.flatten(), Y.flatten()])
transformed = A @ points
fig, axes = plt.subplots(1, 2, figsize=(14, 6))
# Before transformation
ax1 = axes[0]
ax1.scatter(points[0], points[1], alpha=0.3, s=10, color='blue')
setup_ax(ax1, xlim=(-3, 3), ylim=(-3, 3))
ax1.set_title('Before: 2D Space', fontsize=14, fontweight='bold')
# After transformation
ax2 = axes[1]
ax2.scatter(transformed[0], transformed[1], alpha=0.3, s=10, color='red')
setup_ax(ax2, xlim=(-10, 10), ylim=(-10, 10))
ax2.set_title('After: Squished to a Line!', fontsize=14, fontweight='bold')
# Draw the line
t = np.linspace(-10, 10, 100)
line_direction = A @ np.array([1, 0]) # Direction of squished line
ax2.plot(t * line_direction[0], t * line_direction[1], 'g--',
linewidth=2, label='All vectors squished here', alpha=0.7)
ax2.legend(fontsize=11)
plt.tight_layout()
plt.show()
print("\nNotice: Entire 2D plane → single line")
print("This means: Cannot reverse (no inverse exists!)")
visualize_zero_determinant()
4. Column Space and Rank¶
Column Space¶
The column space of matrix \(A\) is the set of all possible outputs \(Ax\).
For a 2×2 matrix:
Column space could be entire 2D plane (full rank)
Or just a line (rank 1)
Or a point (rank 0)
Rank¶
Rank = number of dimensions in the column space
Full rank: Column space = entire output space
Rank deficient: Column space is smaller
Key insight: Rank tells you how many dimensions “survive” the transformation.
Connection to Determinant¶
det(A) ≠ 0 ⟺ full rank ⟺ columns are linearly independent
det(A) = 0 ⟺ rank deficient ⟺ columns are linearly dependent
def visualize_column_space():
"""Visualize column space for different matrices."""
matrices = [
("Full Rank (Rank 2)", np.array([[3, 1], [1, 2]])),
("Rank 1 (Line)", np.array([[2, 4], [1, 2]])),
]
fig, axes = plt.subplots(1, 2, figsize=(14, 6))
for idx, (name, A) in enumerate(matrices):
ax = axes[idx]
# Generate random input vectors
np.random.seed(42)
inputs = np.random.randn(2, 200) * 2
outputs = A @ inputs
# Plot outputs (column space)
ax.scatter(outputs[0], outputs[1], alpha=0.4, s=30, color='purple',
label='Column space')
# Draw column vectors
col1 = A[:, 0]
col2 = A[:, 1]
draw_vector(ax, np.array([0, 0]), col1, color='red', label='Column 1')
draw_vector(ax, np.array([0, 0]), col2, color='blue', label='Column 2')
setup_ax(ax, xlim=(-10, 10), ylim=(-10, 10))
rank = np.linalg.matrix_rank(A)
det = np.linalg.det(A)
ax.set_title(f'{name}\nRank={rank}, det={det:.2f}',
fontsize=14, fontweight='bold')
ax.legend(fontsize=10)
plt.tight_layout()
plt.show()
print("Column Space Interpretation:")
print("- Left: Outputs fill entire 2D plane (rank 2)")
print("- Right: Outputs confined to a line (rank 1)")
print("\nRank = dimensions of column space = 'degrees of freedom'")
visualize_column_space()
5. Null Space (Kernel)¶
Definition¶
The null space (or kernel) of \(A\) is the set of all vectors that land on zero:
Geometric Meaning¶
Null space = all vectors that get “squished” to the origin
For full-rank matrices: null space = {0} (only origin)
For rank-deficient matrices: null space is larger (line, plane, etc.)
Why It Matters¶
When solving \(Ax = v\):
If null space = {0}: unique solution (if it exists)
If null space ≠ {0}: infinite solutions (if any exist)
You can add any null space vector to a solution and still have a solution!
def visualize_null_space():
"""Visualize the null space of a matrix."""
# Matrix with non-trivial null space
A = np.array([[2, 4], [1, 2]])
print("Matrix A:")
print(A)
print(f"\nDeterminant: {np.linalg.det(A):.6f}")
print(f"Rank: {np.linalg.matrix_rank(A)}\n")
# Find null space basis
# For this matrix, null space is span of [-2, 1]
null_basis = np.array([-2, 1])
# Verify it's in null space
result = A @ null_basis
print(f"Null space basis vector: {null_basis}")
print(f"A × null_basis = {result}")
print(f"Indeed maps to zero!\n")
fig, axes = plt.subplots(1, 2, figsize=(14, 6))
# Left: Null space (input side)
ax1 = axes[0]
setup_ax(ax1)
# Draw the null space line
t = np.linspace(-3, 3, 100)
null_line = np.outer(null_basis, t)
ax1.plot(null_line[0], null_line[1], 'g-', linewidth=3,
label='Null space (gets squished to 0)', alpha=0.7)
# Draw specific null vectors
for scalar in [-2, -1, 1, 2]:
vec = scalar * null_basis
draw_vector(ax1, np.array([0, 0]), vec, color='green', width=0.008)
ax1.set_title('Input: Null Space Vectors', fontsize=14, fontweight='bold')
ax1.legend(fontsize=11)
# Right: All map to origin
ax2 = axes[1]
setup_ax(ax2)
# Show that all null vectors → 0
ax2.scatter([0], [0], s=200, color='red', marker='*',
label='All null space vectors land here!', zorder=5)
ax2.set_title('Output: All → Origin', fontsize=14, fontweight='bold')
ax2.legend(fontsize=11)
plt.tight_layout()
plt.show()
print("Key Insight:")
print("- Null space = vectors that disappear (→ 0)")
print("- For rank-deficient matrices, null space ≠ {0}")
print("- Null space dimension + rank = input dimension")
visualize_null_space()
6. Non-Square Matrices¶
Transformations Between Dimensions¶
Matrices don’t have to be square!
3×2 matrix: 2D → 3D
2×3 matrix: 3D → 2D
1×2 matrix: 2D → 1D (number line)
Reading a matrix shape:
Rows = output dimensions
Columns = input dimensions
Example: A 3×2 matrix maps 2D vectors to 3D vectors.
def visualize_nonsquare():
"""Visualize non-square matrix transformations."""
# 3x2 matrix: 2D → 3D
A = np.array([
[2, 0],
[-1, 1],
[-2, 1]
])
print("3×2 Matrix (2D → 3D):")
print(A)
print(f"\nShape: {A.shape} (3 rows, 2 columns)")
print(f"Rank: {np.linalg.matrix_rank(A)}\n")
# Test vectors
test_vectors_2d = np.array([
[1, 0], [0, 1], [1, 1], [2, -1]
]).T
# Transform to 3D
transformed_3d = A @ test_vectors_2d
fig = plt.figure(figsize=(14, 6))
# Left: Input 2D vectors
ax1 = fig.add_subplot(121)
setup_ax(ax1, xlim=(-3, 3), ylim=(-3, 3))
for i in range(test_vectors_2d.shape[1]):
vec = test_vectors_2d[:, i]
draw_vector(ax1, np.array([0, 0]), vec,
color=plt.cm.viridis(i/4), width=0.01)
ax1.set_title('Input: 2D Vectors', fontsize=14, fontweight='bold')
# Right: Output 3D vectors
ax2 = fig.add_subplot(122, projection='3d')
for i in range(transformed_3d.shape[1]):
vec = transformed_3d[:, i]
ax2.quiver(0, 0, 0, vec[0], vec[1], vec[2],
color=plt.cm.viridis(i/4), arrow_length_ratio=0.15, linewidth=2)
# Draw the column space (2D plane in 3D)
col1 = A[:, 0]
col2 = A[:, 1]
# Create mesh on the plane
u = np.linspace(-2, 2, 10)
v = np.linspace(-2, 2, 10)
U, V = np.meshgrid(u, v)
X = col1[0] * U + col2[0] * V
Y = col1[1] * U + col2[1] * V
Z = col1[2] * U + col2[2] * V
ax2.plot_surface(X, Y, Z, alpha=0.3, color='cyan', label='Column space')
ax2.set_xlabel('X')
ax2.set_ylabel('Y')
ax2.set_zlabel('Z')
ax2.set_title('Output: 3D Vectors (on 2D plane)', fontsize=14, fontweight='bold')
plt.tight_layout()
plt.show()
print("Observation:")
print("- 2D inputs → 3D outputs")
print("- But outputs confined to 2D plane in 3D space")
print("- Column space = 2D plane (rank 2)")
print("- Cannot be full rank in 3D (would need 3 columns)")
visualize_nonsquare()
7. Practical Application: Image Compression¶
Low-Rank Approximation¶
One powerful application of rank is image compression:
Images are matrices (pixels)
We can approximate with lower-rank matrices
Store less data, small quality loss
This uses Singular Value Decomposition (SVD), which finds the best low-rank approximation.
def demonstrate_low_rank_approximation():
"""Show low-rank approximation for a simple 'image'."""
# Create a simple pattern matrix
np.random.seed(42)
n = 50
# Create a low-rank structure + noise
u1 = np.sin(np.linspace(0, 2*np.pi, n))
v1 = np.cos(np.linspace(0, 2*np.pi, n))
# Rank-1 component
img = np.outer(u1, v1) * 10
# Add small noise
img += np.random.randn(n, n) * 0.5
# Compute SVD
U, s, Vt = np.linalg.svd(img, full_matrices=False)
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
ranks = [1, 5, 10, 20, 30, 50]
for idx, r in enumerate(ranks):
ax = axes[idx // 3, idx % 3]
# Reconstruct with rank r
img_approx = U[:, :r] @ np.diag(s[:r]) @ Vt[:r, :]
# Calculate compression ratio
original_size = n * n
compressed_size = r * (n + n + 1) # U + s + Vt
ratio = compressed_size / original_size
im = ax.imshow(img_approx, cmap='viridis')
ax.set_title(f'Rank {r}\nStorage: {ratio:.1%}',
fontsize=12, fontweight='bold')
ax.axis('off')
plt.colorbar(im, ax=ax)
plt.tight_layout()
plt.show()
print("Low-Rank Approximation:")
print(f"Original: {n}×{n} = {n*n} values")
print("\nWith rank r approximation:")
print(" Storage = r×(2n + 1) values")
print("\nRank 1: ~4% storage, captures main pattern")
print("Rank 10: ~40% storage, very close to original")
print("\nThis is how JPEG and video compression work!")
demonstrate_low_rank_approximation()
8. Summary and Key Takeaways¶
Main Ideas¶
Linear systems as transformations
\(Ax = v\) means “find \(x\) that lands on \(v\)”
Geometric problem, not just algebra
Matrix inverse
\(A^{-1}\) “rewinds” the transformation
Exists when det(A) ≠ 0
\(A^{-1}A = I\)
Column space
All possible outputs \(Ax\)
Span of column vectors
Represents “where vectors can land”
Rank
Dimensions of column space
Number of linearly independent columns
Tells you “degrees of freedom”
Null space
Vectors that map to zero
Empty (just origin) for full-rank
Larger for rank-deficient matrices
Non-square matrices
Can map between different dimensions
Shape tells you: rows = output, cols = input
Decision Tree for Solving \(Ax = v\)¶
Is det(A) ≠ 0?
├─ YES → Unique solution: x = A⁻¹v
└─ NO → Is v in column space?
├─ YES → Infinite solutions (add null space vectors)
└─ NO → No solution
Fundamental Theorem¶
For an m×n matrix A:
This connects the “surviving dimensions” and “squished dimensions”!
9. Practice Exercises¶
Exercise 1: Check Invertibility¶
For each matrix, determine if it’s invertible. If yes, find the inverse:
a) \(\begin{bmatrix} 3 & 1 \\ 2 & 1 \end{bmatrix}\)
b) \(\begin{bmatrix} 2 & 6 \\ 1 & 3 \end{bmatrix}\)
c) \(\begin{bmatrix} 1 & 0 \\ 0 & 1 \end{bmatrix}\)
# Your code here
def check_invertibility(A):
det = np.linalg.det(A)
print(f"Matrix:\n{A}")
print(f"Determinant: {det:.4f}")
if abs(det) > 1e-10:
A_inv = np.linalg.inv(A)
print(f"Invertible! Inverse:\n{A_inv}")
# Verify
print(f"Verification A⁻¹A:\n{A_inv @ A}")
else:
print("Not invertible (det = 0)")
print("-" * 40)
# Test the matrices
matrices = [
np.array([[3, 1], [2, 1]]),
np.array([[2, 6], [1, 3]]),
np.array([[1, 0], [0, 1]])
]
for A in matrices:
check_invertibility(A)
Exercise 2: Find Rank and Null Space¶
For matrix \(A = \begin{bmatrix} 1 & 2 & 3 \\ 2 & 4 & 6 \\ 1 & 2 & 3 \end{bmatrix}\):
a) Find the rank
b) Find the dimension of the null space
c) Find a basis for the null space
# Your code here
A = np.array([[1, 2, 3],
[2, 4, 6],
[1, 2, 3]])
rank = np.linalg.matrix_rank(A)
n_cols = A.shape[1]
null_dim = n_cols - rank
print(f"Matrix A:\n{A}")
print(f"\nRank: {rank}")
print(f"Null space dimension: {null_dim}")
print(f"\nVerify: rank + null_dim = {rank + null_dim} (should equal {n_cols})")
# Find null space using SVD
U, s, Vt = np.linalg.svd(A)
null_space = Vt[rank:].T
print(f"\nNull space basis vectors:\n{null_space}")
# Verify
print(f"\nVerification A × null_vector = {A @ null_space[:, 0]}")
Exercise 3: Non-Square Matrix¶
Given \(A = \begin{bmatrix} 1 & 2 \\ 3 & 4 \\ 5 & 6 \end{bmatrix}\) (3×2 matrix):
a) What are the dimensions of input and output spaces?
b) What is the rank?
c) Can this matrix be invertible? Why or why not?
d) Visualize the column space
# Your code here
A = np.array([[1, 2],
[3, 4],
[5, 6]])
print(f"Matrix shape: {A.shape}")
print(f"Input dimension: {A.shape[1]} (2D)")
print(f"Output dimension: {A.shape[0]} (3D)")
print(f"\nRank: {np.linalg.matrix_rank(A)}")
print(f"\nInvertible? No - not square!")
print(f"Also, maps 2D → 3D, so output is at most 2D plane")
Further Reading¶
Gaussian elimination: Method for computing inverses and solving systems
Row echelon form: Standard form for analyzing matrices
Singular Value Decomposition (SVD): Powerful generalization of eigendecomposition
Least squares: Best approximate solution when exact solution doesn’t exist
Next: Dot products and duality - a surprising connection between transformations and geometry!