Applying Kernels for Image Segmentation¶
The goal of this project is to try and understand and replicate Python's Pillow Filter2D method by creating and applying my own filter. This project is a result of collaborating with members of PDX's Data Science group to better understand Neural Nets, Convolutions, Kernels, and Padding.
This project seeks to understand kernels and to practice applying kernels as masks to the image.
I build my code from the resources provided below:
Image Kernel Visualizaiton
Sobel Derivation Explained
Gaussian Differences
Breaking Symmetry
Libraries:
from PIL import Image
import numpy as np
import os
from shutil import copyfile
from collections import Counter
from scipy.ndimage import gaussian_filter
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import cv2
import numpy as np
%matplotlib inline
1. Pillow's Filter Method¶
impath='small/training/ADE_train_00000341.jpg'
image = mpimg.imread(impath)
# Read in the image
image_mask = mpimg.imread('small/training/ADE_train_00000341_seg.png')
plt.imshow(image)
plt.title('Original Image')
plt.show()
# Convert to grayscale for filtering
gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
plt.imshow(gray, cmap='gray')
# Create a custom kernel
# 3x3 array for edge detection
sobel_y = np.array([[ -1, -2, -1],
[ 0, 0, 0],
[ 1, 2, 1]])
# Filter the image using filter2D, which has inputs: (grayscale image, bit-depth, kernel)
filtered_image = cv2.filter2D(gray, -1, sobel_y)
plt.imshow(filtered_image, cmap='gray')
plt.title("Image with Sobel Filter")
plt.show()
The picture above shows the application of a sobel filter to the image using PIL's filter2D method.
2. My Filter Method¶
# Kernel to matrix:
# Applies a 3x3 Mask to the image array
def apply_kernel(m,k):
new = m.copy()
for r in range(253):
for c in range(253):
face = np.array([m[r][c],m[r][c+1],m[r][c+2],
m[r+1][c],m[r+1][c+1],m[r+1][c+2],
m[r+2][c],m[r+2][c+1],m[r+2][c+2]]).reshape(3,3)
new_val = int(np.sum(face*k))
if new_val<=0:
new_val = 0
elif new_val>=255:
new_val = 255
new[r][c] = new_val
return new
# Apply Gaussian to matrix
def apply_blur(m,s):
return gaussian_filter(m,sigma=s)
# Show the matrix as image
def show_image(im,name):
plt.title(name)
plt.imshow(Image.fromarray(im))
plt.show()
# Code for Fixing Resize to keep scale
# https://stackoverflow.com/questions/44231209/resize-rectangular-image-to-square-keeping-ratio-and-fill-background-with-black/44231784
def make_square(im, min_size=255, fill_color=(0, 0, 0, 0)):
x, y = im.size
size = max(min_size, x, y)
new_im = Image.new('RGBA', (size, size), fill_color)
new_im.paste(im, int(((size - x) / 2), int((size - y) / 2)))
return new_im
# Convert image to grid
def create_grid(p):
im = Image.open(p).convert('L').resize((255,255))
return np.array(im)
# Practice final output
def segment_spaces(m):
test_1 = apply_blur(apply_kernel(m,right_sobel),10)
test_2 = apply_blur(apply_kernel(m,left_sobel),10)
test_3 = apply_blur(apply_kernel(m,bottom_sobel),10)
test_4 = apply_blur(apply_kernel(m,top_sobel),10)
show_image((test_1+test_2+test_3+test_4)/4,'Sharpen after Sobel Top')
# Kernels and their 3x3 dimensions
right_sobel = np.array(
[-1, 0, 1,
-2, 0, 2,
-1, 0, 1]).reshape(3,3)
left_sobel = np.array(
[ 1, 0,-1,
2, 0,-2,
1, 0,-1]).reshape(3,3)
bottom_sobel = np.array(
[-1,-2,-1,
0, 0, 0,
1, 2, 1]).reshape(3,3)
top_sobel = np.array(
[ 1, 2, 1,
0, 0, 0,
-1,-2,-1]).reshape(3,3)
sharpen = np.array(
[ 0,-1, 0,
-1, 5,-1,
0,-1, 0]).reshape(3,3)
emboss = np.array(
[-2, 1, 0,
-1, 1, 1,
0, 1, 2]).reshape(3,3)
# Matrix view and Image view of subsection
impath='small/training/ADE_train_00000341.jpg'
example = create_grid(impath)
print(example[10:30,10:20])
plt.imshow(example[10:30,10:20])
plt.show()
final_grids = []
final_blur = []
for mask in [right_sobel, left_sobel,bottom_sobel,top_sobel]:
output_grid = apply_kernel(grid, mask)
final_grids.append(output_grid)
blurred1 = gaussian_filter(output_grid, sigma=10)
blurred2 = gaussian_filter(output_grid, sigma=15)
final_blur.append(blurred2-blurred1)
show_image(output_grid,mask)
show_image(np.abs(blurred2+blurred1)/2,'blurred')
show_image(final_grids[3],'Sobel Top')
kernel_two = apply_kernel(final_grids[3],sharpen)
show_image(kernel_two,'Sharpen after Sobel Top')
print('---')
show_image(final_grids[3],'Sobel Top')
kernel_two = apply_kernel(final_grids[3],emboss)
show_image(kernel_two,'Emboss after Sobel Top')
As you can see from the images above. I was able to successfuly apply filters to the image. I was also able to apply a combination of filters to an image.
Testing Multiple Images:¶
names = ['ADE_train_00000341','ADE_train_00002021','ADE_train_00002074']
for n in names:
impath = f"small/training/{n}.jpg"
plt.imshow(Image.open(impath).resize((255,255)))
plt.show()
segment_spaces(create_grid(impath))
The above images appear to mostly work, however that is an issue with my differences of gaussians method in the final photo. I need to explore this to figure out why there are extreme edges in the final photo.
Conclusion¶
I was able to better understand how Convulutions work by practicing my application of a number of different filters to real images by using a 3x3 mask.
Next Steps:
- Add Padding
- Explore the "Sharpen after Sobel Top" outlier image
- Build a model that connects the premade segmented image to my altered image.
Misc¶
This is a collection of other mask/filter tests I did with my code.
plt.imshow((final_grids[0]+final_grids[1]+final_grids[2]+final_grids[3])/4)
Difference of Gaussians¶
Incomplete code (keeping images):¶
plt.imshow(Image.fromarray(np.abs(blurred2-blurred1)))
plt.show()
(blurred2-blurred1).max()
plt.imshow(Image.fromarray(np.abs(blurred2+blurred1)/2))
plt.show()
plt.imshow(image_mask)
plt.show()
