Run this notebook: Open in Colab Open in Kaggle

Chapter 7: Inverse matrices, column space and null space

1. The Dot Product: Two Definitions

Algebraic Definition

For vectors \(v = \begin{bmatrix} v_1 \\ v_2 \end{bmatrix}\) and \(w = \begin{bmatrix} w_1 \\ w_2 \end{bmatrix}\):

\[ v \cdot w = v_1 w_1 + v_2 w_2 \]

Multiply corresponding coordinates, then add.

Geometric Definition

Project \(w\) onto \(v\), then multiply:

\[ v \cdot w = \|v\| \times (\text{length of projection of } w \text{ onto } v) \]

Or equivalently:

\[ v \cdot w = \|v\| \|w\| \cos(\theta) \]

where \(\theta\) is the angle between them.

The Mystery

How are these the same?!

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import FancyArrowPatch
from matplotlib.patches import Rectangle
import seaborn as sns

sns.set_style("whitegrid")
plt.rcParams['figure.figsize'] = (10, 8)

def draw_vector(ax, origin, vector, color='blue', label=None, width=0.01):
    """Draw a vector as an arrow."""
    arrow = FancyArrowPatch(
        origin, origin + vector,
        mutation_scale=20, lw=2.5, arrowstyle='-|>', color=color, label=label
    )
    ax.add_patch(arrow)

def setup_ax(ax, xlim=(-1, 5), ylim=(-1, 5)):
    """Setup 2D axis."""
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)
    ax.set_aspect('equal')
    ax.axhline(y=0, color='k', linewidth=0.5)
    ax.axvline(x=0, color='k', linewidth=0.5)
    ax.grid(True, alpha=0.3)
    ax.set_xlabel('x', fontsize=12)
    ax.set_ylabel('y', fontsize=12)

def visualize_dot_product():
    """Visualize both definitions of dot product."""
    v = np.array([3, 1])
    w = np.array([1, 3])
    
    # Algebraic
    dot_algebraic = v[0] * w[0] + v[1] * w[1]
    
    # Geometric
    v_len = np.linalg.norm(v)
    w_len = np.linalg.norm(w)
    
    # Project w onto v
    projection_length = (w @ v) / v_len
    dot_geometric = v_len * projection_length
    
    # Projection vector
    v_unit = v / v_len
    projection = projection_length * v_unit
    
    fig, axes = plt.subplots(1, 2, figsize=(14, 6))
    
    # Left: Algebraic
    ax1 = axes[0]
    setup_ax(ax1)
    draw_vector(ax1, np.array([0, 0]), v, color='red', label=f'v = {v}')
    draw_vector(ax1, np.array([0, 0]), w, color='blue', label=f'w = {w}')
    ax1.set_title(f'Algebraic: v·w = {v[0]}×{w[0]} + {v[1]}×{w[1]} = {dot_algebraic}', 
                 fontsize=13, fontweight='bold')
    ax1.legend(fontsize=11)
    
    # Right: Geometric
    ax2 = axes[1]
    setup_ax(ax2)
    draw_vector(ax2, np.array([0, 0]), v, color='red', label=f'v (length={v_len:.2f})')
    draw_vector(ax2, np.array([0, 0]), w, color='blue', label=f'w')
    draw_vector(ax2, np.array([0, 0]), projection, color='green', 
               label=f'projection (length={projection_length:.2f})')
    
    # Draw perpendicular drop line
    ax2.plot([w[0], projection[0]], [w[1], projection[1]], 
            'g--', linewidth=1.5, alpha=0.7, label='perpendicular')
    
    ax2.set_title(f'Geometric: {v_len:.2f} × {projection_length:.2f} = {dot_geometric:.2f}', 
                 fontsize=13, fontweight='bold')
    ax2.legend(fontsize=10)
    
    plt.tight_layout()
    plt.show()
    
    print(f"Algebraic: {dot_algebraic}")
    print(f"Geometric: {dot_geometric:.4f}")
    print(f"\nThey match! ✓")

visualize_dot_product()

2. Sign of the Dot Product

What the Sign Tells You

  • Positive: Vectors point in generally the same direction (angle < 90°)

  • Zero: Vectors are perpendicular (angle = 90°)

  • Negative: Vectors point in opposite directions (angle > 90°)

This is because projection length can be negative!

def visualize_dot_product_signs():
    """Show positive, zero, and negative dot products."""
    v = np.array([3, 1])
    
    # Three different w vectors
    cases = [
        ("Positive (acute)", np.array([2, 2])),
        ("Zero (perpendicular)", np.array([-1, 3])),
        ("Negative (obtuse)", np.array([-2, 1]))
    ]
    
    fig, axes = plt.subplots(1, 3, figsize=(16, 5))
    
    for idx, (name, w) in enumerate(cases):
        ax = axes[idx]
        setup_ax(ax, xlim=(-3, 4), ylim=(-1, 4))
        
        # Compute dot product
        dot = v @ w
        
        # Projection
        v_len = np.linalg.norm(v)
        v_unit = v / v_len
        proj_len = (w @ v) / v_len
        projection = proj_len * v_unit
        
        # Draw vectors
        draw_vector(ax, np.array([0, 0]), v, color='red', label='v')
        draw_vector(ax, np.array([0, 0]), w, color='blue', label='w')
        
        if abs(dot) > 0.1:  # Not perpendicular
            draw_vector(ax, np.array([0, 0]), projection, color='green', label='projection')
            ax.plot([w[0], projection[0]], [w[1], projection[1]], 
                   'g--', linewidth=1.5, alpha=0.7)
        
        # Angle
        angle = np.arccos(dot / (np.linalg.norm(v) * np.linalg.norm(w)))
        angle_deg = np.degrees(angle)
        
        ax.set_title(f'{name}\nv·w = {dot:.2f}, θ = {angle_deg:.1f}°', 
                    fontsize=12, fontweight='bold')
        ax.legend(fontsize=10)
    
    plt.tight_layout()
    plt.show()
    
    print("Key Insight:")
    print("- Acute angle (< 90°) → positive dot product")
    print("- Right angle (= 90°) → zero dot product")
    print("- Obtuse angle (> 90°) → negative dot product")

visualize_dot_product_signs()

3. Commutativity: Order Doesn’t Matter

The Puzzle

The geometric definition seems asymmetric:

  • Project \(w\) onto \(v\)

But we could also:

  • Project \(v\) onto \(w\)

How can both give the same result?

The Answer

When vectors have the same length, it’s symmetric (mirror image).

When you scale one vector:

  • Scaling \(v\) by 2 doubles the result in both interpretations

  • Either way: \((2v) \cdot w = 2(v \cdot w)\)

So commutativity holds!

def visualize_commutativity():
    """Show why v·w = w·v."""
    v = np.array([3, 1])
    w = np.array([1, 2.5])
    
    # Both ways
    v_len = np.linalg.norm(v)
    w_len = np.linalg.norm(w)
    
    # w projected onto v
    proj_w_on_v_len = (w @ v) / v_len
    proj_w_on_v = proj_w_on_v_len * (v / v_len)
    result1 = v_len * proj_w_on_v_len
    
    # v projected onto w
    proj_v_on_w_len = (v @ w) / w_len
    proj_v_on_w = proj_v_on_w_len * (w / w_len)
    result2 = w_len * proj_v_on_w_len
    
    fig, axes = plt.subplots(1, 2, figsize=(14, 6))
    
    # Left: w onto v
    ax1 = axes[0]
    setup_ax(ax1)
    draw_vector(ax1, np.array([0, 0]), v, color='red', label=f'v (||v||={v_len:.2f})')
    draw_vector(ax1, np.array([0, 0]), w, color='blue', label='w')
    draw_vector(ax1, np.array([0, 0]), proj_w_on_v, color='green', 
               label=f'projection length={proj_w_on_v_len:.2f}')
    ax1.plot([w[0], proj_w_on_v[0]], [w[1], proj_w_on_v[1]], 'g--', linewidth=1.5, alpha=0.7)
    ax1.set_title(f'Project w onto v\nResult = {v_len:.2f} × {proj_w_on_v_len:.2f} = {result1:.2f}', 
                 fontsize=12, fontweight='bold')
    ax1.legend(fontsize=10)
    
    # Right: v onto w
    ax2 = axes[1]
    setup_ax(ax2)
    draw_vector(ax2, np.array([0, 0]), w, color='blue', label=f'w (||w||={w_len:.2f})')
    draw_vector(ax2, np.array([0, 0]), v, color='red', label='v')
    draw_vector(ax2, np.array([0, 0]), proj_v_on_w, color='purple', 
               label=f'projection length={proj_v_on_w_len:.2f}')
    ax2.plot([v[0], proj_v_on_w[0]], [v[1], proj_v_on_w[1]], 'm--', linewidth=1.5, alpha=0.7)
    ax2.set_title(f'Project v onto w\nResult = {w_len:.2f} × {proj_v_on_w_len:.2f} = {result2:.2f}', 
                 fontsize=12, fontweight='bold')
    ax2.legend(fontsize=10)
    
    plt.tight_layout()
    plt.show()
    
    print(f"v·w = {result1:.4f}")
    print(f"w·v = {result2:.4f}")
    print(f"\nSame result! Commutativity holds ✓")

visualize_commutativity()

4. Duality: 1×2 Matrices and 2D Vectors

Transformations to 1D

A linear transformation from 2D to 1D (the number line) can be represented by a 1×2 matrix:

\[\begin{split} \begin{bmatrix} u_1 & u_2 \end{bmatrix} \begin{bmatrix} x \\ y \end{bmatrix} = u_1 x + u_2 y \end{split}\]

This is exactly the dot product formula!

The Duality

There’s a one-to-one correspondence:

\[ \text{2D vector } u \leftrightarrow \text{Linear transformation } (v \mapsto u \cdot v) \]

Every 2D vector defines a unique linear transformation to 1D, and vice versa!

Geometric Meaning

Think of a diagonal number line in 2D space. Projecting vectors onto this line is a linear transformation!

def visualize_duality():
    """Show the duality between vectors and 1×2 matrices."""
    # The special vector u
    u = np.array([2, 1])
    
    # Create test vectors
    test_vectors = np.array([
        [1, 0], [0, 1], [1, 1], [2, -1], [-1, 2]
    ]).T
    
    # Apply transformation (dot product with u)
    results = u @ test_vectors
    
    fig, axes = plt.subplots(1, 2, figsize=(14, 6))
    
    # Left: 2D space with test vectors
    ax1 = axes[0]
    setup_ax(ax1, xlim=(-2, 3), ylim=(-2, 3))
    
    # Draw the "number line" direction (u)
    t = np.linspace(-2, 3, 100)
    u_norm = u / np.linalg.norm(u)
    ax1.plot(t * u_norm[0], t * u_norm[1], 'g--', linewidth=2, 
            label='projection line (u direction)', alpha=0.7)
    
    draw_vector(ax1, np.array([0, 0]), u, color='green', label=f'u = {u}')
    
    # Draw test vectors
    for i in range(test_vectors.shape[1]):
        vec = test_vectors[:, i]
        draw_vector(ax1, np.array([0, 0]), vec, 
                   color=plt.cm.viridis(i/5), width=0.008)
    
    ax1.set_title('Input: 2D Vectors', fontsize=14, fontweight='bold')
    ax1.legend(fontsize=11)
    
    # Right: 1D number line with results
    ax2 = axes[1]
    ax2.axhline(y=0, color='k', linewidth=0.5)
    ax2.set_xlim(-3, 5)
    ax2.set_ylim(-0.5, 0.5)
    ax2.set_xlabel('Number Line', fontsize=12)
    ax2.set_yticks([])
    ax2.grid(True, alpha=0.3)
    
    # Plot results on number line
    for i, result in enumerate(results):
        ax2.scatter([result], [0], s=150, color=plt.cm.viridis(i/5), zorder=5)
        ax2.text(result, 0.15, f'{result:.1f}', ha='center', fontsize=10)
    
    ax2.set_title('Output: Numbers (u·v for each v)', fontsize=14, fontweight='bold')
    
    plt.tight_layout()
    plt.show()
    
    print("Duality Insight:")
    print(f"Vector u = {u} defines a transformation:")
    print(f"  v → u·v (project onto u's direction)")
    print(f"\nAs a 1×2 matrix: [{u[0]} {u[1]}]")
    print(f"\nEvery vector ↔ unique transformation to 1D!")

visualize_duality()

5. Why Projection = Dot Product

The Full Picture

Imagine a diagonal number line through space. To project a vector onto this line:

  1. Find the unit vector \(\hat{u}\) pointing along the line

  2. Drop perpendicular from your vector to the line

  3. Read off the number where it lands

This projection operation:

  • Is a linear transformation (2D → 1D)

  • Can be represented by a 1×2 matrix \([u_1 \quad u_2]\)

  • Corresponds to the 2D vector \(\begin{bmatrix} u_1 \\ u_2 \end{bmatrix}\)

Applying this transformation is exactly the dot product!

\[\begin{split} \begin{bmatrix} u_1 & u_2 \end{bmatrix} \begin{bmatrix} v_1 \\ v_2 \end{bmatrix} = u \cdot v \end{split}\]
def demonstrate_projection_transformation():
    """Full demonstration of projection as transformation."""
    # Unit vector defining the number line
    u_hat = np.array([3, 1])
    u_hat = u_hat / np.linalg.norm(u_hat)
    
    # Test vector
    v = np.array([2, 3])
    
    # Projection (as a number)
    projection_number = u_hat @ v
    
    # Projection (as a vector in 2D)
    projection_vector = projection_number * u_hat
    
    fig = plt.figure(figsize=(12, 10))
    
    # Main plot
    ax = fig.add_subplot(111)
    setup_ax(ax, xlim=(-1, 4), ylim=(-1, 4))
    
    # Draw the diagonal number line
    t = np.linspace(-1, 4, 100)
    ax.plot(t * u_hat[0], t * u_hat[1], 'g-', linewidth=3, 
           label='Diagonal number line', alpha=0.5)
    
    # Mark numbers on the line
    for num in range(-1, 5):
        pos = num * u_hat
        ax.scatter([pos[0]], [pos[1]], s=50, color='green', zorder=5)
        ax.text(pos[0] + 0.2, pos[1] + 0.2, str(num), fontsize=11)
    
    # Draw u_hat
    draw_vector(ax, np.array([0, 0]), u_hat, color='green', 
               label=f'û = unit vector = [{u_hat[0]:.2f}, {u_hat[1]:.2f}]')
    
    # Draw v
    draw_vector(ax, np.array([0, 0]), v, color='blue', label=f'v = {v}')
    
    # Draw projection
    draw_vector(ax, np.array([0, 0]), projection_vector, color='red', 
               label=f'projection = {projection_number:.2f}')
    
    # Draw perpendicular
    ax.plot([v[0], projection_vector[0]], [v[1], projection_vector[1]], 
           'r--', linewidth=2, alpha=0.7, label='perpendicular drop')
    
    # Highlight where it lands
    ax.scatter([projection_vector[0]], [projection_vector[1]], 
              s=200, color='red', marker='*', zorder=10, 
              label=f'Lands at {projection_number:.2f}')
    
    ax.set_title('Projection as Linear Transformation\nû·v = "Where does v land on the û number line?"', 
                fontsize=14, fontweight='bold')
    ax.legend(fontsize=10, loc='upper left')
    
    plt.tight_layout()
    plt.show()
    
    print("Complete Picture:")
    print(f"1. Unit vector û = {u_hat}")
    print(f"2. As 1×2 matrix: [{u_hat[0]:.3f}  {u_hat[1]:.3f}]")
    print(f"3. Test vector v = {v}")
    print(f"4. Transformation: v → û·v = {projection_number:.3f}")
    print(f"\nThis IS the dot product!")

demonstrate_projection_transformation()

6. Applications of Dot Products

1. Testing Perpendicularity

Two vectors are perpendicular ⟺ \(v \cdot w = 0\)

2. Finding Angles

\[ \cos(\theta) = \frac{v \cdot w}{\|v\| \|w\|} \]

3. Measuring Similarity

  • Large positive dot product → vectors point same direction

  • Used in machine learning (cosine similarity)

4. Projections

Project \(v\) onto \(w\):

\[ \text{proj}_w(v) = \frac{v \cdot w}{\|w\|^2} w \]
def demonstrate_applications():
    """Show practical applications of dot products."""
    
    # Application 1: Find perpendicular vector
    v = np.array([3, 2])
    # Perpendicular is [-v[1], v[0]]
    v_perp = np.array([-v[1], v[0]])
    
    print("Application 1: Perpendicularity Test")
    print(f"v = {v}")
    print(f"v_perp = {v_perp}")
    print(f"v · v_perp = {v @ v_perp}")
    print(f"Perpendicular? {abs(v @ v_perp) < 1e-10}\n")
    
    # Application 2: Angle between vectors
    w = np.array([1, 4])
    cos_theta = (v @ w) / (np.linalg.norm(v) * np.linalg.norm(w))
    theta = np.arccos(cos_theta)
    theta_deg = np.degrees(theta)
    
    print("Application 2: Finding Angles")
    print(f"v = {v}, w = {w}")
    print(f"cos(θ) = {cos_theta:.4f}")
    print(f"θ = {theta_deg:.2f}°\n")
    
    # Application 3: Cosine similarity (ML)
    # Simulate document vectors
    doc1 = np.array([3, 4, 1])  # word frequencies
    doc2 = np.array([2, 5, 0])
    doc3 = np.array([0, 1, 10])
    
    # Normalize for cosine similarity
    def cosine_sim(a, b):
        return (a @ b) / (np.linalg.norm(a) * np.linalg.norm(b))
    
    sim_12 = cosine_sim(doc1, doc2)
    sim_13 = cosine_sim(doc1, doc3)
    
    print("Application 3: Document Similarity (ML)")
    print(f"Doc1 = {doc1} (word counts)")
    print(f"Doc2 = {doc2}")
    print(f"Doc3 = {doc3}")
    print(f"\nSimilarity(Doc1, Doc2) = {sim_12:.4f}")
    print(f"Similarity(Doc1, Doc3) = {sim_13:.4f}")
    print(f"Doc1 is more similar to Doc2! ✓\n")
    
    # Application 4: Component in a direction
    force = np.array([10, 5])  # Force vector
    direction = np.array([1, 0])  # Horizontal direction
    
    component = force @ direction  # Since direction is unit
    
    print("Application 4: Force Component")
    print(f"Force = {force} N")
    print(f"Direction = {direction} (horizontal)")
    print(f"Horizontal component = {component} N")

demonstrate_applications()

7. Extending to Higher Dimensions

Everything generalizes beautifully to 3D, 4D, or any dimension:

\[ v \cdot w = v_1 w_1 + v_2 w_2 + \cdots + v_n w_n \]

Still means:

  • Project \(w\) onto \(v\)

  • Test perpendicularity

  • Measure angles

  • Linear transformation to 1D

The duality also holds: n-dimensional vectors ↔ 1×n matrices

def demonstrate_high_dimensional():
    """Show dot products in higher dimensions."""
    # 5-dimensional vectors
    v = np.array([1, 2, 3, 4, 5])
    w = np.array([5, 4, 3, 2, 1])
    
    dot = v @ w
    v_norm = np.linalg.norm(v)
    w_norm = np.linalg.norm(w)
    
    cos_theta = dot / (v_norm * w_norm)
    theta = np.arccos(cos_theta)
    theta_deg = np.degrees(theta)
    
    print("5-Dimensional Example:")
    print(f"v = {v}")
    print(f"w = {w}")
    print(f"\nDot product: {dot}")
    print(f"||v|| = {v_norm:.4f}")
    print(f"||w|| = {w_norm:.4f}")
    print(f"\nAngle: {theta_deg:.2f}°")
    print(f"\nAll the same concepts apply!")
    
    # Test perpendicularity in higher dimensions
    print("\n" + "="*50)
    print("Perpendicular vectors in 4D:")
    a = np.array([1, 2, 3, 4])
    # Create perpendicular by making dot product = 0
    # If a = [1,2,3,4], then b = [4,-2,0,0] is perpendicular
    b = np.array([4, -2, 0, 0])
    
    print(f"a = {a}")
    print(f"b = {b}")
    print(f"a·b = {a @ b}")
    print(f"Perpendicular? {abs(a @ b) < 1e-10}")

demonstrate_high_dimensional()

8. Summary and Key Takeaways

The Big Ideas

  1. Two equivalent definitions

    • Algebraic: multiply coordinates and sum

    • Geometric: projection and scaling

  2. Sign interpretation

    • Positive: same direction (< 90°)

    • Zero: perpendicular (= 90°)

    • Negative: opposite direction (> 90°)

  3. Commutativity

    • \(v \cdot w = w \cdot v\)

    • Works from both projection perspectives

  4. Duality (the deep insight!)

    • Every vector ↔ unique linear transformation to 1D

    • 2D vector \(u\) ↔ 1×2 matrix \([u_1 \quad u_2]\)

    • Dot product = matrix-vector multiplication

  5. Geometric meaning

    • Projection onto a diagonal number line

    • “Where does \(w\) land on the \(v\) number line?”

Formula Summary

\[ v \cdot w = \sum v_i w_i = \|v\| \|w\| \cos(\theta) \]

Practical Applications

  • ✅ Test perpendicularity (\(v \cdot w = 0\))

  • ✅ Find angles between vectors

  • ✅ Compute projections

  • ✅ Measure similarity (ML, data science)

  • ✅ Decompose vectors into components

The Profound Connection

Dot products bridge three worlds:

  1. Algebra (coordinate multiplication)

  2. Geometry (projections)

  3. Transformations (linear maps to 1D)

This is duality in action!

9. Practice Exercises

Exercise 1: Compute and Interpret

For \(v = [4, 1]\) and \(w = [-1, 3]\):

a) Compute \(v \cdot w\)

b) Are they pointing in the same or opposite direction?

c) Find the angle between them

# Your code here
v = np.array([4, 1])
w = np.array([-1, 3])

dot = v @ w
print(f"a) v·w = {dot}")
print(f"\nb) Sign is {'positive' if dot > 0 else 'negative' if dot < 0 else 'zero'}")
print(f"   So they point in {'the same' if dot > 0 else 'opposite' if dot < 0 else 'perpendicular'} direction")

cos_theta = dot / (np.linalg.norm(v) * np.linalg.norm(w))
theta_deg = np.degrees(np.arccos(cos_theta))
print(f"\nc) Angle = {theta_deg:.2f}°")

Exercise 2: Find Perpendicular

Given \(v = [5, 2]\), find a vector perpendicular to it.

# Your code here
v = np.array([5, 2])
v_perp = np.array([-v[1], v[0]])  # Rotate 90 degrees

print(f"v = {v}")
print(f"Perpendicular = {v_perp}")
print(f"\nVerification: v·perp = {v @ v_perp}")
print(f"Is perpendicular? {abs(v @ v_perp) < 1e-10}")

Exercise 3: Projection

Project \(v = [3, 4]\) onto \(w = [1, 0]\) (horizontal direction).

What is the projection vector?

# Your code here
v = np.array([3, 4])
w = np.array([1, 0])

# Projection formula: proj_w(v) = (v·w / ||w||²) * w
proj_v_on_w = ((v @ w) / (w @ w)) * w

print(f"v = {v}")
print(f"w = {w} (horizontal)")
print(f"\nProjection of v onto w = {proj_v_on_w}")
print(f"\nMeaning: The horizontal component of v is {proj_v_on_w[0]}")

Further Reading

  • Cross products: Another product for vectors (next notebook!)

  • Inner product spaces: Abstract generalization of dot products

  • Gram-Schmidt process: Orthogonalization using projections

  • Dual spaces: The mathematical formalization of duality

Next: Cross products - finding perpendicular vectors and measuring oriented area!