CNN Filter Visualization

This notebook demonstrates how Convolutional Neural Network (CNN) filters work by visualizing various filter operations on images.

import numpy as np
import matplotlib.pyplot as plt
from scipy import signal
from PIL import Image
import requests
from io import BytesIO
import warnings
warnings.filterwarnings('ignore')

# Set style for better visualizations
plt.style.use('seaborn-v0_8-darkgrid')
plt.rcParams['figure.figsize'] = (15, 10)
plt.rcParams['font.size'] = 12

1. Understanding Convolution Operations

def apply_filter(image, kernel):
    """Apply a convolution filter to an image."""
    if len(image.shape) == 3:
        # Convert to grayscale if RGB
        image = np.dot(image[...,:3], [0.2989, 0.5870, 0.1140])
    
    # Apply convolution
    filtered = signal.convolve2d(image, kernel, mode='same', boundary='symm')
    return filtered

def normalize_image(image):
    """Normalize image to 0-255 range."""
    img_min, img_max = image.min(), image.max()
    if img_max - img_min > 0:
        return ((image - img_min) / (img_max - img_min) * 255).astype(np.uint8)
    return image.astype(np.uint8)

2. Create Sample Image

# Create a synthetic image with various patterns
def create_sample_image(size=200):
    """Create a synthetic image with geometric patterns."""
    image = np.zeros((size, size))
    
    # Add vertical lines
    for i in range(20, size, 40):
        image[:, i:i+3] = 255
    
    # Add horizontal lines
    for i in range(20, size, 40):
        image[i:i+3, :] = 128
    
    # Add diagonal lines
    for i in range(0, size-20, 30):
        for j in range(3):
            if i+j < size:
                np.fill_diagonal(image[i+j:, :], 200)
    
    # Add a square
    image[60:100, 60:100] = 180
    
    # Add a circle
    center = (150, 150)
    radius = 30
    y, x = np.ogrid[:size, :size]
    mask = (x - center[0])**2 + (y - center[1])**2 <= radius**2
    image[mask] = 255
    
    return image

# Create sample image
sample_image = create_sample_image()

plt.figure(figsize=(6, 6))
plt.imshow(sample_image, cmap='gray')
plt.title('Sample Input Image')
plt.axis('off')
plt.colorbar()
plt.show()

3. Define Common CNN Filters

# Define various filters
filters = {
    'Edge Detection (Sobel X)': np.array([[-1, 0, 1],
                                          [-2, 0, 2],
                                          [-1, 0, 1]]),
    
    'Edge Detection (Sobel Y)': np.array([[-1, -2, -1],
                                          [ 0,  0,  0],
                                          [ 1,  2,  1]]),
    
    'Sharpen': np.array([[ 0, -1,  0],
                         [-1,  5, -1],
                         [ 0, -1,  0]]),
    
    'Blur (Box)': np.ones((5, 5)) / 25,
    
    'Gaussian Blur': np.array([[1, 2, 1],
                               [2, 4, 2],
                               [1, 2, 1]]) / 16,
    
    'Emboss': np.array([[-2, -1, 0],
                        [-1,  1, 1],
                        [ 0,  1, 2]]),
    
    'Outline': np.array([[-1, -1, -1],
                         [-1,  8, -1],
                         [-1, -1, -1]]),
    
    'Identity': np.array([[0, 0, 0],
                         [0, 1, 0],
                         [0, 0, 0]]),
    
    'Top Edge': np.array([[ 1,  2,  1],
                         [ 0,  0,  0],
                         [-1, -2, -1]]),
    
    'Right Edge': np.array([[-1, 0, 1],
                           [-2, 0, 2],
                           [-1, 0, 1]]),
    
    'Bottom Edge': np.array([[-1, -2, -1],
                            [ 0,  0,  0],
                            [ 1,  2,  1]]),
    
    'Left Edge': np.array([[1, 0, -1],
                          [2, 0, -2],
                          [1, 0, -1]])
}

4. Visualize Filter Effects

# Apply and visualize all filters
fig, axes = plt.subplots(4, 4, figsize=(20, 20))
axes = axes.flatten()

# Show original image
axes[0].imshow(sample_image, cmap='gray')
axes[0].set_title('Original Image', fontsize=14, fontweight='bold')
axes[0].axis('off')

# Apply each filter
for idx, (filter_name, kernel) in enumerate(filters.items(), 1):
    if idx < len(axes):
        filtered_img = apply_filter(sample_image, kernel)
        axes[idx].imshow(filtered_img, cmap='gray')
        axes[idx].set_title(filter_name, fontsize=12)
        axes[idx].axis('off')

# Hide unused subplots
for idx in range(len(filters) + 1, len(axes)):
    axes[idx].axis('off')

plt.suptitle('CNN Filter Visualization', fontsize=16, fontweight='bold', y=1.02)
plt.tight_layout()
plt.show()

5. Visualize Filter Kernels

# Visualize the filter kernels themselves
fig, axes = plt.subplots(3, 4, figsize=(16, 12))
axes = axes.flatten()

for idx, (filter_name, kernel) in enumerate(filters.items()):
    if idx < len(axes):
        im = axes[idx].imshow(kernel, cmap='RdBu_r', vmin=-2, vmax=2)
        axes[idx].set_title(filter_name, fontsize=11)
        
        # Add text annotations for kernel values
        for i in range(kernel.shape[0]):
            for j in range(kernel.shape[1]):
                text = axes[idx].text(j, i, f'{kernel[i, j]:.2f}',
                                     ha="center", va="center", color="black", fontsize=8)
        
        axes[idx].set_xticks([])
        axes[idx].set_yticks([])

# Hide unused subplots
for idx in range(len(filters), len(axes)):
    axes[idx].axis('off')

plt.suptitle('Filter Kernel Visualizations', fontsize=16, fontweight='bold')
plt.tight_layout()
plt.show()

6. Interactive Filter Combination

# Combine multiple filters
def combine_filters(image, filter_list):
    """Apply multiple filters sequentially."""
    result = image.copy()
    for filter_name in filter_list:
        if filter_name in filters:
            result = apply_filter(result, filters[filter_name])
    return result

# Example combinations
combinations = [
    (['Gaussian Blur', 'Edge Detection (Sobel X)'], 'Blur → Sobel X'),
    (['Edge Detection (Sobel X)', 'Edge Detection (Sobel Y)'], 'Sobel X → Sobel Y'),
    (['Sharpen', 'Emboss'], 'Sharpen → Emboss'),
    (['Blur (Box)', 'Outline'], 'Blur → Outline')
]

fig, axes = plt.subplots(2, 4, figsize=(20, 10))

for idx, (filter_combo, title) in enumerate(combinations):
    row = idx // 2
    col = (idx % 2) * 2
    
    # Show original
    axes[row, col].imshow(sample_image, cmap='gray')
    axes[row, col].set_title('Original', fontsize=12)
    axes[row, col].axis('off')
    
    # Show filtered
    combined = combine_filters(sample_image, filter_combo)
    axes[row, col + 1].imshow(combined, cmap='gray')
    axes[row, col + 1].set_title(title, fontsize=12, fontweight='bold')
    axes[row, col + 1].axis('off')

plt.suptitle('Sequential Filter Combinations', fontsize=16, fontweight='bold')
plt.tight_layout()
plt.show()

7. Feature Maps Visualization

# Simulate multiple feature maps like in a CNN layer
def create_feature_maps(image, num_filters=8):
    """Create random filters and apply them to generate feature maps."""
    feature_maps = []
    filter_kernels = []
    
    # Use some predefined filters
    predefined = list(filters.values())[:min(num_filters, len(filters))]
    
    for i in range(num_filters):
        if i < len(predefined):
            kernel = predefined[i]
        else:
            # Create random filter
            kernel = np.random.randn(3, 3)
            kernel = kernel / np.sum(np.abs(kernel))  # Normalize
        
        filter_kernels.append(kernel)
        feature_map = apply_filter(image, kernel)
        feature_maps.append(feature_map)
    
    return feature_maps, filter_kernels

# Generate feature maps
feature_maps, kernels = create_feature_maps(sample_image, num_filters=8)

# Visualize feature maps
fig, axes = plt.subplots(2, 4, figsize=(20, 10))
axes = axes.flatten()

for idx, (fmap, kernel) in enumerate(zip(feature_maps, kernels)):
    if idx < len(axes):
        axes[idx].imshow(fmap, cmap='gray')
        axes[idx].set_title(f'Feature Map {idx+1}', fontsize=12)
        axes[idx].axis('off')

plt.suptitle('CNN Feature Maps (First Layer)', fontsize=16, fontweight='bold')
plt.tight_layout()
plt.show()

8. Activation Functions on Feature Maps

# Apply different activation functions
def relu(x):
    return np.maximum(0, x)

def sigmoid(x):
    return 1 / (1 + np.exp(-x/255))  # Scale for visualization

def tanh_activation(x):
    return np.tanh(x/255)  # Scale for visualization

# Apply edge detection filter
edge_map = apply_filter(sample_image, filters['Edge Detection (Sobel X)'])

# Apply different activations
activations = {
    'Original Feature Map': edge_map,
    'ReLU Activation': relu(edge_map),
    'Sigmoid Activation': sigmoid(edge_map) * 255,
    'Tanh Activation': tanh_activation(edge_map) * 127 + 128
}

fig, axes = plt.subplots(1, 4, figsize=(20, 5))

for idx, (name, activated) in enumerate(activations.items()):
    axes[idx].imshow(activated, cmap='gray')
    axes[idx].set_title(name, fontsize=12)
    axes[idx].axis('off')

plt.suptitle('Activation Functions Applied to Feature Maps', fontsize=16, fontweight='bold')
plt.tight_layout()
plt.show()

9. Pooling Operations

def max_pooling(image, pool_size=2):
    """Apply max pooling to an image."""
    h, w = image.shape
    new_h, new_w = h // pool_size, w // pool_size
    pooled = np.zeros((new_h, new_w))
    
    for i in range(new_h):
        for j in range(new_w):
            pooled[i, j] = np.max(image[i*pool_size:(i+1)*pool_size, 
                                       j*pool_size:(j+1)*pool_size])
    return pooled

def avg_pooling(image, pool_size=2):
    """Apply average pooling to an image."""
    h, w = image.shape
    new_h, new_w = h // pool_size, w // pool_size
    pooled = np.zeros((new_h, new_w))
    
    for i in range(new_h):
        for j in range(new_w):
            pooled[i, j] = np.mean(image[i*pool_size:(i+1)*pool_size, 
                                        j*pool_size:(j+1)*pool_size])
    return pooled

# Apply pooling to edge-detected image
edge_map = apply_filter(sample_image, filters['Outline'])
edge_map = relu(edge_map)  # Apply ReLU first

# Different pooling sizes
pooling_results = {
    'Original (200x200)': edge_map,
    'Max Pool 2x2 (100x100)': max_pooling(edge_map, 2),
    'Max Pool 4x4 (50x50)': max_pooling(edge_map, 4),
    'Avg Pool 2x2 (100x100)': avg_pooling(edge_map, 2)
}

fig, axes = plt.subplots(1, 4, figsize=(20, 5))

for idx, (name, pooled) in enumerate(pooling_results.items()):
    axes[idx].imshow(pooled, cmap='gray')
    axes[idx].set_title(f'{name}', fontsize=12)
    axes[idx].axis('off')

plt.suptitle('Pooling Operations in CNNs', fontsize=16, fontweight='bold')
plt.tight_layout()
plt.show()

10. Complete CNN Pipeline Simulation

# Simulate a complete CNN forward pass
def cnn_forward_pass(image):
    """Simulate a CNN forward pass with multiple layers."""
    results = {'Input': image}
    
    # Layer 1: Convolution + ReLU
    conv1 = apply_filter(image, filters['Edge Detection (Sobel X)'])
    relu1 = relu(conv1)
    results['Conv1 + ReLU'] = relu1
    
    # Layer 1: Max Pooling
    pool1 = max_pooling(relu1, 2)
    results['MaxPool1'] = pool1
    
    # Layer 2: Convolution + ReLU
    conv2 = apply_filter(pool1, filters['Sharpen'])
    relu2 = relu(conv2)
    results['Conv2 + ReLU'] = relu2
    
    # Layer 2: Max Pooling
    pool2 = max_pooling(relu2, 2)
    results['MaxPool2'] = pool2
    
    # Layer 3: Convolution + ReLU
    conv3 = apply_filter(pool2, filters['Outline'])
    relu3 = relu(conv3)
    results['Conv3 + ReLU'] = relu3
    
    # Final pooling
    final_pool = max_pooling(relu3, 2)
    results['Final Output'] = final_pool
    
    return results

# Run the simulation
pipeline_results = cnn_forward_pass(sample_image)

# Visualize the pipeline
fig, axes = plt.subplots(2, 4, figsize=(20, 10))
axes = axes.flatten()

for idx, (layer_name, output) in enumerate(pipeline_results.items()):
    if idx < len(axes):
        axes[idx].imshow(output, cmap='gray')
        axes[idx].set_title(f'{layer_name}\nShape: {output.shape}', fontsize=11)
        axes[idx].axis('off')

# Hide unused subplots
for idx in range(len(pipeline_results), len(axes)):
    axes[idx].axis('off')

plt.suptitle('Complete CNN Pipeline Visualization', fontsize=16, fontweight='bold')
plt.tight_layout()
plt.show()

11. Custom Filter Design

# Let users experiment with custom filters
custom_filters = {
    'Diagonal Edge (45°)': np.array([[2, 1, 0],
                                     [1, 0, -1],
                                     [0, -1, -2]]),
    
    'Diagonal Edge (135°)': np.array([[0, 1, 2],
                                      [-1, 0, 1],
                                      [-2, -1, 0]]),
    
    'Cross Pattern': np.array([[0, 1, 0],
                               [1, -4, 1],
                               [0, 1, 0]]),
    
    'Corner Detection': np.array([[-1, -1, -1],
                                  [-1, 8, -1],
                                  [-1, -1, -1]]),
    
    'Ridge Detection': np.array([[0, -1, 0],
                                 [-1, 4, -1],
                                 [0, -1, 0]]),
    
    'Motion Blur': np.array([[0, 0, 0, 0, 1],
                            [0, 0, 0, 1, 0],
                            [0, 0, 1, 0, 0],
                            [0, 1, 0, 0, 0],
                            [1, 0, 0, 0, 0]]) / 5
}

# Apply custom filters
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
axes = axes.flatten()

for idx, (filter_name, kernel) in enumerate(custom_filters.items()):
    if idx < len(axes):
        filtered = apply_filter(sample_image, kernel)
        axes[idx].imshow(filtered, cmap='gray')
        axes[idx].set_title(filter_name, fontsize=12)
        axes[idx].axis('off')

plt.suptitle('Custom Filter Designs', fontsize=16, fontweight='bold')
plt.tight_layout()
plt.show()

12. Filter Response Heatmap

# Create a heatmap showing filter responses
def compute_filter_responses(image, filter_dict):
    """Compute the mean absolute response for each filter."""
    responses = {}
    for name, kernel in filter_dict.items():
        filtered = apply_filter(image, kernel)
        responses[name] = np.mean(np.abs(filtered))
    return responses

# Compute responses for all filters
all_filters = {**filters, **custom_filters}
responses = compute_filter_responses(sample_image, all_filters)

# Sort by response strength
sorted_responses = dict(sorted(responses.items(), key=lambda x: x[1], reverse=True))

# Create bar plot
plt.figure(figsize=(12, 8))
names = list(sorted_responses.keys())
values = list(sorted_responses.values())

colors = plt.cm.viridis(np.linspace(0, 1, len(names)))
bars = plt.barh(range(len(names)), values, color=colors)

plt.yticks(range(len(names)), names)
plt.xlabel('Mean Absolute Response', fontsize=12)
plt.title('Filter Response Strength Analysis', fontsize=14, fontweight='bold')
plt.grid(axis='x', alpha=0.3)

# Add value labels
for bar, value in zip(bars, values):
    plt.text(value, bar.get_y() + bar.get_height()/2, f'{value:.1f}', 
             ha='left', va='center', fontsize=9)

plt.tight_layout()
plt.show()

Summary

This notebook demonstrated:

1. Convolution Operations: How filters are applied to images using convolution
2. Common CNN Filters: Edge detection, sharpening, blurring, and other filters
3. Filter Kernels: Visualization of the actual filter weights
4. Sequential Filtering: How filters can be combined for complex effects
5. Feature Maps: Multiple filters creating different feature representations
6. Activation Functions: ReLU, Sigmoid, and Tanh applied to feature maps
7. Pooling Operations: Max and average pooling for dimensionality reduction
8. Complete CNN Pipeline: Simulating multiple layers of a CNN
9. Custom Filters: Designing specialized filters for specific patterns
10. Filter Response Analysis: Quantifying filter effectiveness

These visualizations help understand how CNNs extract features from images through learned filters.