Run this notebook: Open in Colab Open in Kaggle 
# Setup
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns

sns.set_style('whitegrid')
np.set_printoptions(precision=3, suppress=True)

3.1 Norms

Definition 3.1 (Norm)

A norm on a vector space V is a function ||·|| : V → ℝ that satisfies:

  1. Positive definite: ||x|| ≥ 0 and ||x|| = 0 ⟺ x = 0

  2. Triangle inequality: ||x + y|| ≤ ||x|| + ||y||

  3. Homogeneity: ||λx|| = |λ| ||x||

Common Norms:

Manhattan (L₁): \(||x||_1 = \sum_{i=1}^n |x_i|\)

Euclidean (L₂): \(||x||_2 = \sqrt{\sum_{i=1}^n x_i^2}\)

Max (L∞): \(||x||_\infty = \max_i |x_i|\)

# Different norms

x = np.array([3, 4])

print("Vector x =", x)
print("\nNorms:")

# L1 norm (Manhattan)
l1_norm = np.linalg.norm(x, ord=1)
print(f"L₁ (Manhattan): ||x||₁ = {l1_norm}")

# L2 norm (Euclidean)
l2_norm = np.linalg.norm(x, ord=2)
print(f"L₂ (Euclidean): ||x||₂ = {l2_norm}")

# L-infinity norm (Max)
linf_norm = np.linalg.norm(x, ord=np.inf)
print(f"L∞ (Max):       ||x||∞ = {linf_norm}")

# Manual calculation for L2
l2_manual = np.sqrt(np.sum(x**2))
print(f"\nL₂ manual: √(3² + 4²) = √25 = {l2_manual}")

# Visualize unit circles for different norms

fig, axes = plt.subplots(1, 3, figsize=(15, 5))

# Generate points on unit circles
theta = np.linspace(0, 2*np.pi, 1000)

# L1 norm (diamond)
t = np.linspace(0, 2*np.pi, 1000)
x_l1 = np.sign(np.cos(t)) * np.abs(np.cos(t))
y_l1 = np.sign(np.sin(t)) * np.abs(np.sin(t))

axes[0].plot(x_l1, y_l1, 'b-', linewidth=2)
axes[0].set_aspect('equal')
axes[0].grid(True, alpha=0.3)
axes[0].set_title('L₁ Norm (Manhattan)\n||x||₁ = 1', fontsize=12)
axes[0].set_xlabel('x₁')
axes[0].set_ylabel('x₂')
axes[0].axhline(0, color='black', linewidth=0.5)
axes[0].axvline(0, color='black', linewidth=0.5)

# L2 norm (circle)
x_l2 = np.cos(theta)
y_l2 = np.sin(theta)

axes[1].plot(x_l2, y_l2, 'r-', linewidth=2)
axes[1].set_aspect('equal')
axes[1].grid(True, alpha=0.3)
axes[1].set_title('L₂ Norm (Euclidean)\n||x||₂ = 1', fontsize=12)
axes[1].set_xlabel('x₁')
axes[1].set_ylabel('x₂')
axes[1].axhline(0, color='black', linewidth=0.5)
axes[1].axvline(0, color='black', linewidth=0.5)

# L-infinity norm (square)
x_linf = np.array([-1, 1, 1, -1, -1])
y_linf = np.array([-1, -1, 1, 1, -1])

axes[2].plot(x_linf, y_linf, 'g-', linewidth=2)
axes[2].set_aspect('equal')
axes[2].grid(True, alpha=0.3)
axes[2].set_title('L∞ Norm (Max)\n||x||∞ = 1', fontsize=12)
axes[2].set_xlabel('x₁')
axes[2].set_ylabel('x₂')
axes[2].axhline(0, color='black', linewidth=0.5)
axes[2].axvline(0, color='black', linewidth=0.5)

plt.tight_layout()
plt.show()

print("Unit circles show all points with norm = 1")
print("Different norms → different notions of 'distance'")

3.2 Inner Products

Definition 3.2 (Inner Product)

An inner product on V is a mapping ⟨·,·⟩ : V × V → ℝ that satisfies:

  1. Symmetric: ⟨x, y⟩ = ⟨y, x⟩

  2. Linear: ⟨λx + z, y⟩ = λ⟨x, y⟩ + ⟨z, y⟩

  3. Positive definite: ⟨x, x⟩ ≥ 0 and ⟨x, x⟩ = 0 ⟺ x = 0

Dot Product (Standard Inner Product):

\[⟨x, y⟩ = x^T y = \sum_{i=1}^n x_i y_i\]
# Inner product (dot product)

x = np.array([1, 2, 3])
y = np.array([4, 5, 6])

print("Vectors:")
print(f"x = {x}")
print(f"y = {y}")

# Dot product
dot_product = np.dot(x, y)
print(f"\n⟨x, y⟩ = x·y = {dot_product}")

# Manual calculation
manual = x[0]*y[0] + x[1]*y[1] + x[2]*y[2]
print(f"Manual: {x[0]}×{y[0]} + {x[1]}×{y[1]} + {x[2]}×{y[2]} = {manual}")

# Alternative notation
print(f"\nAlternative: x @ y = {x @ y}")

# Properties
print("\n" + "="*50)
print("Properties:")
print(f"Symmetric: ⟨x,y⟩ = {np.dot(x,y)}, ⟨y,x⟩ = {np.dot(y,x)}")
print(f"⟨x,x⟩ = {np.dot(x,x)} (always ≥ 0)")
print(f"||x||₂² = ⟨x,x⟩ = {np.dot(x,x)} = {np.linalg.norm(x)**2:.3f}")

# General inner product with positive definite matrix
# ⟨x, y⟩_A = x^T A y where A is positive definite

x = np.array([1, 2])
y = np.array([3, 4])

# Standard inner product (A = I)
I = np.eye(2)
inner_std = x @ I @ y
print("Standard inner product (A = I):")
print(f"⟨x, y⟩ = {inner_std}")

# Custom inner product with A = [[2, 0], [0, 3]]
A = np.array([[2, 0],
              [0, 3]])
inner_custom = x @ A @ y
print("\nCustom inner product (A = [[2,0],[0,3]]):")
print(f"⟨x, y⟩_A = {inner_custom}")

# This weights coordinates differently!
print("\nInterpretation: Custom inner products weight coordinates differently")

3.3 Lengths and Distances

Length from Inner Product:

\[||x|| = \sqrt{⟨x, x⟩}\]

Distance:

\[d(x, y) = ||x - y|| = \sqrt{⟨x-y, x-y⟩}\]

Cauchy-Schwarz Inequality:

\[|⟨x, y⟩| \leq ||x|| \cdot ||y||\]
# Distances between vectors

x = np.array([1, 2, 3])
y = np.array([4, 6, 8])

print("Vectors:")
print(f"x = {x}")
print(f"y = {y}")

# Euclidean distance
diff = y - x
euclidean_dist = np.linalg.norm(diff)
print(f"\nEuclidean distance: d(x,y) = ||y-x||₂ = {euclidean_dist:.3f}")

# Manhattan distance
manhattan_dist = np.linalg.norm(diff, ord=1)
print(f"Manhattan distance: d(x,y) = ||y-x||₁ = {manhattan_dist:.3f}")

# Manual Euclidean
manual = np.sqrt(np.sum(diff**2))
print(f"\nManual: √((4-1)² + (6-2)² + (8-3)²) = √(9+16+25) = √50 = {manual:.3f}")

# Verify Cauchy-Schwarz inequality
print("\n" + "="*50)
print("Cauchy-Schwarz Inequality: |⟨x,y⟩| ≤ ||x|| ||y||")
lhs = np.abs(np.dot(x, y))
rhs = np.linalg.norm(x) * np.linalg.norm(y)
print(f"LHS: |⟨x,y⟩| = {lhs:.3f}")
print(f"RHS: ||x|| ||y|| = {rhs:.3f}")
print(f"Inequality holds: {lhs <= rhs}")

# Visualize distances in 2D

x = np.array([1, 1])
y = np.array([4, 5])

plt.figure(figsize=(10, 8))

# Plot points
plt.plot(*x, 'ro', markersize=12, label='x')
plt.plot(*y, 'bo', markersize=12, label='y')

# Euclidean distance (straight line)
plt.plot([x[0], y[0]], [x[1], y[1]], 'g-', linewidth=3, label=f'Euclidean: {np.linalg.norm(y-x):.2f}')

# Manhattan distance (L-shaped)
plt.plot([x[0], y[0]], [x[1], x[1]], 'r--', linewidth=2, alpha=0.7)
plt.plot([y[0], y[0]], [x[1], y[1]], 'r--', linewidth=2, alpha=0.7, label=f'Manhattan: {np.linalg.norm(y-x, ord=1):.2f}')

# Annotate points
plt.text(x[0]-0.3, x[1]-0.3, f'x=({x[0]},{x[1]})', fontsize=11)
plt.text(y[0]+0.1, y[1]+0.1, f'y=({y[0]},{y[1]})', fontsize=11)

plt.grid(True, alpha=0.3)
plt.legend(fontsize=11)
plt.title('Different Distance Measures', fontsize=14)
plt.xlabel('x₁', fontsize=12)
plt.ylabel('x₂', fontsize=12)
plt.axis('equal')
plt.tight_layout()
plt.show()

3.4 Angles and Orthogonality

Angle Between Vectors:

\[\cos(\omega) = \frac{⟨x, y⟩}{||x|| \cdot ||y||}\]

Definition 3.4 (Orthogonality)

Two vectors x and y are orthogonal (perpendicular) if:

\[⟨x, y⟩ = 0\]

Notation: x ⊥ y

# Angle between vectors

def angle_between(x, y, degrees=True):
    """Compute angle between two vectors."""
    cos_angle = np.dot(x, y) / (np.linalg.norm(x) * np.linalg.norm(y))
    # Clamp to avoid numerical issues
    cos_angle = np.clip(cos_angle, -1, 1)
    angle_rad = np.arccos(cos_angle)
    return np.degrees(angle_rad) if degrees else angle_rad

# Example 1: 45-degree angle
x = np.array([1, 0])
y = np.array([1, 1])

angle = angle_between(x, y)
print("Example 1:")
print(f"x = {x}")
print(f"y = {y}")
print(f"Angle: {angle:.2f}°")

# Example 2: Orthogonal vectors
x = np.array([1, 0, 0])
y = np.array([0, 1, 0])

print("\nExample 2 (orthogonal):")
print(f"x = {x}")
print(f"y = {y}")
print(f"⟨x, y⟩ = {np.dot(x, y)}")
print(f"Angle: {angle_between(x, y):.2f}°")
print(f"Orthogonal: {np.dot(x, y) == 0}")

# Example 3: Opposite directions
x = np.array([1, 2])
y = -x  # Opposite direction

print("\nExample 3 (opposite):")
print(f"x = {x}")
print(f"y = {y}")
print(f"Angle: {angle_between(x, y):.2f}°")

# Visualize angles

fig, axes = plt.subplots(1, 3, figsize=(15, 5))

# Case 1: Acute angle
x1 = np.array([2, 0])
y1 = np.array([1.5, 1.5])
angle1 = angle_between(x1, y1)

axes[0].quiver(0, 0, x1[0], x1[1], angles='xy', scale_units='xy', scale=1, 
               color='r', width=0.015, label='x')
axes[0].quiver(0, 0, y1[0], y1[1], angles='xy', scale_units='xy', scale=1, 
               color='b', width=0.015, label='y')
axes[0].set_xlim(-0.5, 3)
axes[0].set_ylim(-0.5, 3)
axes[0].set_aspect('equal')
axes[0].grid(True, alpha=0.3)
axes[0].legend()
axes[0].set_title(f'Acute Angle\n{angle1:.1f}°', fontsize=12)

# Case 2: Orthogonal (90°)
x2 = np.array([2, 0])
y2 = np.array([0, 2])

axes[1].quiver(0, 0, x2[0], x2[1], angles='xy', scale_units='xy', scale=1, 
               color='r', width=0.015, label='x')
axes[1].quiver(0, 0, y2[0], y2[1], angles='xy', scale_units='xy', scale=1, 
               color='b', width=0.015, label='y')
# Draw right angle marker
square = plt.Rectangle((0, 0), 0.3, 0.3, fill=False, edgecolor='green', linewidth=2)
axes[1].add_patch(square)
axes[1].set_xlim(-0.5, 3)
axes[1].set_ylim(-0.5, 3)
axes[1].set_aspect('equal')
axes[1].grid(True, alpha=0.3)
axes[1].legend()
axes[1].set_title('Orthogonal\n90.0° (⟨x,y⟩=0)', fontsize=12)

# Case 3: Obtuse angle
x3 = np.array([2, 0])
y3 = np.array([-1, 1.5])
angle3 = angle_between(x3, y3)

axes[2].quiver(0, 0, x3[0], x3[1], angles='xy', scale_units='xy', scale=1, 
               color='r', width=0.015, label='x')
axes[2].quiver(0, 0, y3[0], y3[1], angles='xy', scale_units='xy', scale=1, 
               color='b', width=0.015, label='y')
axes[2].set_xlim(-2, 3)
axes[2].set_ylim(-0.5, 3)
axes[2].set_aspect('equal')
axes[2].grid(True, alpha=0.3)
axes[2].legend()
axes[2].set_title(f'Obtuse Angle\n{angle3:.1f}°', fontsize=12)

plt.tight_layout()
plt.show()

3.5 Orthonormal Basis

Definition 3.5 (Orthonormal Basis)

A basis {b₁, …, bₙ} is orthonormal if:

  1. Orthogonal: ⟨bᵢ, bⱼ⟩ = 0 for i ≠ j

  2. Normalized: ||bᵢ|| = 1 for all i

Matrix form: If B = [b₁ … bₙ], then B^T B = I

Benefits:

  • Simple coordinate transformations

  • Numerically stable computations

# Gram-Schmidt orthogonalization
# Convert arbitrary basis to orthonormal basis

def gram_schmidt(vectors):
    """
    Gram-Schmidt orthogonalization.
    Returns orthonormal basis.
    """
    basis = []
    
    for v in vectors:
        # Subtract projections onto previous basis vectors
        w = v.copy()
        for b in basis:
            w = w - np.dot(v, b) * b
        
        # Normalize
        if np.linalg.norm(w) > 1e-10:
            w = w / np.linalg.norm(w)
            basis.append(w)
    
    return np.array(basis)

# Start with arbitrary basis
v1 = np.array([1.0, 1.0, 0.0])
v2 = np.array([1.0, 0.0, 1.0])
v3 = np.array([0.0, 1.0, 1.0])

print("Original vectors:")
print(f"v₁ = {v1}")
print(f"v₂ = {v2}")
print(f"v₃ = {v3}")

# Apply Gram-Schmidt
orthonormal_basis = gram_schmidt([v1, v2, v3])

print("\nOrthonormal basis:")
for i, b in enumerate(orthonormal_basis):
    print(f"b₁ = {b}")

# Verify orthonormality
print("\n" + "="*50)
print("Verification:")
B = orthonormal_basis

# Check orthogonality
print("\nOrthogonality (should be 0):")
print(f"⟨b₀, b₁⟩ = {np.dot(B[0], B[1]):.6f}")
print(f"⟨b₀, b₂⟩ = {np.dot(B[0], B[2]):.6f}")
print(f"⟨b₁, b₂⟩ = {np.dot(B[1], B[2]):.6f}")

# Check normalization
print("\nNormalization (should be 1):")
for i, b in enumerate(B):
    print(f"||b₁|| = {np.linalg.norm(b):.6f}")

# Check B^T B = I
print("\nB^T B (should be identity):")
print(B @ B.T)

3.8 Orthogonal Projections

Definition 3.10 (Orthogonal Projection)

The orthogonal projection of x onto a subspace U is the closest point in U to x.

Projection onto a Vector:

\[\text{proj}_b(x) = \frac{⟨x, b⟩}{⟨b, b⟩} b = \frac{⟨x, b⟩}{||b||^2} b\]

Projection Matrix:

If B is orthonormal: \(P = BB^T\)

# Projection onto a vector

def project_onto_vector(x, b):
    """
    Project x onto b.
    """
    return (np.dot(x, b) / np.dot(b, b)) * b

# Example
x = np.array([3, 4])
b = np.array([1, 0])  # Project onto x-axis

proj_x = project_onto_vector(x, b)

print("Vector x =", x)
print("Basis vector b =", b)
print(f"\nProjection of x onto b: {proj_x}")

# Perpendicular component
perp = x - proj_x
print(f"Perpendicular component: {perp}")

# Verify orthogonality
print(f"\nVerify ⟨proj, perp⟩ = {np.dot(proj_x, perp):.10f} (should be 0)")

# Verify reconstruction
reconstructed = proj_x + perp
print(f"Reconstruction: proj + perp = {reconstructed}")
print(f"Original: x = {x}")

# Visualize projection

x = np.array([3, 2])
b = np.array([4, 1])

# Compute projection
proj_x = project_onto_vector(x, b)
perp = x - proj_x

plt.figure(figsize=(10, 8))

# Original vector
plt.quiver(0, 0, x[0], x[1], angles='xy', scale_units='xy', scale=1, 
           color='blue', width=0.012, label='x (original)', zorder=3)

# Basis vector
plt.quiver(0, 0, b[0], b[1], angles='xy', scale_units='xy', scale=1, 
           color='red', width=0.012, label='b (basis)', zorder=3)

# Projection
plt.quiver(0, 0, proj_x[0], proj_x[1], angles='xy', scale_units='xy', scale=1, 
           color='green', width=0.015, label='proj_b(x)', zorder=3)

# Perpendicular component
plt.quiver(proj_x[0], proj_x[1], perp[0], perp[1], angles='xy', scale_units='xy', scale=1, 
           color='orange', width=0.012, label='perpendicular', linestyle='--', zorder=3)

# Dotted line from x to projection
plt.plot([x[0], proj_x[0]], [x[1], proj_x[1]], 'k--', alpha=0.3, linewidth=1)

# Extend basis line
t = np.linspace(-0.5, 1.5, 100)
line = np.outer(t, b)
plt.plot(line[:, 0], line[:, 1], 'r--', alpha=0.3, linewidth=1)

plt.xlim(-1, 5)
plt.ylim(-1, 3)
plt.grid(True, alpha=0.3)
plt.legend(fontsize=11)
plt.title('Orthogonal Projection', fontsize=14)
plt.xlabel('x₁', fontsize=12)
plt.ylabel('x₂', fontsize=12)
plt.axis('equal')
plt.tight_layout()
plt.show()

print(f"x = {x}")
print(f"proj_b(x) = {proj_x}")
print(f"perpendicular = {perp}")
print(f"\nAngle between b and perp: {angle_between(b, perp):.1f}° (should be 90°)")

# Projection onto a plane (2D subspace in 3D)

def project_onto_plane(x, basis_vectors):
    """
    Project x onto plane spanned by basis_vectors.
    """
    # Orthonormalize basis
    B = gram_schmidt(basis_vectors).T
    
    # Projection matrix P = BB^T
    P = B @ B.T
    
    return P @ x

# 3D example: project onto xy-plane
x = np.array([1, 2, 3])

# xy-plane is spanned by e1=[1,0,0] and e2=[0,1,0]
e1 = np.array([1.0, 0.0, 0.0])
e2 = np.array([0.0, 1.0, 0.0])

proj = project_onto_plane(x, [e1, e2])

print("3D vector x =", x)
print(f"Projection onto xy-plane: {proj}")
print(f"(Should zero out z-coordinate)")

# Visualize in 3D
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111, projection='3d')

# Original vector
ax.quiver(0, 0, 0, x[0], x[1], x[2], color='blue', arrow_length_ratio=0.1, linewidth=3, label='x')

# Projection
ax.quiver(0, 0, 0, proj[0], proj[1], proj[2], color='green', arrow_length_ratio=0.1, linewidth=3, label='proj (on xy-plane)')

# Perpendicular
perp = x - proj
ax.quiver(proj[0], proj[1], proj[2], perp[0], perp[1], perp[2], 
          color='orange', arrow_length_ratio=0.1, linewidth=2, linestyle='--', label='perpendicular')

# xy-plane
xx, yy = np.meshgrid(range(-1, 3), range(-1, 4))
zz = np.zeros_like(xx)
ax.plot_surface(xx, yy, zz, alpha=0.2, color='gray')

ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')
ax.legend()
ax.set_title('Projection onto xy-plane')
plt.show()

3.9 Rotations

Definition 3.11 (Rotation Matrix)

A matrix R is a rotation if:

  1. R is orthogonal: R^T R = I

  2. det® = 1 (preserves orientation)

2D Rotation (by angle θ):

\[\begin{split}R(\theta) = \begin{bmatrix} \cos\theta & -\sin\theta \\ \sin\theta & \cos\theta \end{bmatrix}\end{split}\]

Properties:

  • Preserves lengths: ||Rx|| = ||x||

  • Preserves angles: ⟨Rx, Ry⟩ = ⟨x, y⟩

# 2D rotations

def rotation_matrix_2d(theta):
    """
    2D rotation matrix for angle theta (in radians).
    """
    c = np.cos(theta)
    s = np.sin(theta)
    return np.array([[c, -s],
                     [s,  c]])

# Rotate by 45 degrees
theta = np.pi / 4  # 45 degrees
R = rotation_matrix_2d(theta)

print(f"Rotation matrix (45°):")
print(R)

# Apply to vector
x = np.array([1, 0])
x_rotated = R @ x

print(f"\nOriginal: x = {x}")
print(f"Rotated:  Rx = {x_rotated}")

# Verify properties
print("\n" + "="*50)
print("Properties:")
print(f"R^T R = I: {np.allclose(R.T @ R, np.eye(2))}")
print(f"det(R) = 1: {np.isclose(np.linalg.det(R), 1)}")
print(f"||x|| = {np.linalg.norm(x):.3f}")
print(f"||Rx|| = {np.linalg.norm(x_rotated):.3f} (preserved)")

# Visualize rotation sequence

fig, axes = plt.subplots(1, 3, figsize=(15, 5))

angles = [30, 90, 180]  # degrees
x = np.array([2, 1])

for ax, angle_deg in zip(axes, angles):
    angle_rad = np.radians(angle_deg)
    R = rotation_matrix_2d(angle_rad)
    x_rot = R @ x
    
    # Original vector
    ax.quiver(0, 0, x[0], x[1], angles='xy', scale_units='xy', scale=1, 
              color='blue', width=0.015, label='original', zorder=3)
    
    # Rotated vector
    ax.quiver(0, 0, x_rot[0], x_rot[1], angles='xy', scale_units='xy', scale=1, 
              color='red', width=0.015, label='rotated', zorder=3)
    
    # Arc showing rotation
    arc_angles = np.linspace(0, angle_rad, 50)
    arc_r = 0.5
    arc_x = arc_r * np.cos(arc_angles + np.arctan2(x[1], x[0]))
    arc_y = arc_r * np.sin(arc_angles + np.arctan2(x[1], x[0]))
    ax.plot(arc_x, arc_y, 'g-', linewidth=2, alpha=0.5)
    
    ax.set_xlim(-3, 3)
    ax.set_ylim(-3, 3)
    ax.set_aspect('equal')
    ax.grid(True, alpha=0.3)
    ax.legend()
    ax.set_title(f'Rotation by {angle_deg}°', fontsize=12)
    ax.axhline(0, color='black', linewidth=0.5)
    ax.axvline(0, color='black', linewidth=0.5)

plt.tight_layout()
plt.show()

# 3D rotation around z-axis

def rotation_matrix_3d_z(theta):
    """
    3D rotation around z-axis.
    """
    c = np.cos(theta)
    s = np.sin(theta)
    return np.array([[c, -s, 0],
                     [s,  c, 0],
                     [0,  0, 1]])

# Rotate around z-axis
theta = np.pi / 3  # 60 degrees
R_z = rotation_matrix_3d_z(theta)

print("3D Rotation matrix (60° around z-axis):")
print(R_z)

# Apply to vector
x = np.array([1, 0, 1])
x_rot = R_z @ x

print(f"\nOriginal: {x}")
print(f"Rotated:  {x_rot}")
print(f"\nNote: z-component unchanged = {x[2]:.3f} → {x_rot[2]:.3f}")

Summary

Key Concepts from Chapter 3:

  1. Norms: Different ways to measure vector lengths (L₁, L₂, L∞)

  2. Inner Products: Generalize dot product, define angles

  3. Distances: Measure similarity between vectors

  4. Angles & Orthogonality: Perpendicularity is key in ML

  5. Orthonormal Basis: Simplifies computations

  6. Projections: Finding closest points in subspaces

  7. Rotations: Preserve structure while transforming

ML Applications:

  • Distance metrics: k-NN, clustering (Chapter 11)

  • Orthogonal projections: PCA (Chapter 10), least squares

  • Orthonormal bases: Numerical stability, efficient computation

  • Angles: Cosine similarity in NLP, recommendation systems

  • Norms: Regularization (L₁, L₂), measuring errors

Next Steps:

  • Chapter 4: Matrix Decompositions (SVD, eigendecomposition)

  • Chapter 5: Vector Calculus (gradients for optimization)

  • Chapter 10: PCA (applies projections)

Practice: Implement k-NN using different distance metrics!