Tag Archives: Caffe

Deep Dreams with Caffe on Ubuntu 16.04

Googlenet Model

Download the bvlc_googlenet.caffemodel from https://github.com/BVLC/caffe/tree/master/models/bvlc_googlenet

and put it in
caffe/models/bvlc_googlenet/

PIP

sudo apt install python-pip

1	sudo apt install python-pip

IPython, scipy, Jupyter, protobuf, scikit-image

Always install in the user space with --user

pip install --user IPython==5.0 scipy Jupyter protobuf scikit-image

1	pip install --user IPython==5.0 scipy Jupyter protobuf scikit-image

Running jupyter notebook

export PYTHONPATH=/home/behnam/usr/python:$PYTHONPATH
jupyter notebook

1 2	export PYTHONPATH=/home/behnam/usr/python:$PYTHONPATH jupyter notebook

open a new notebook and paste the following into it and correct the "model_path" and

img = np.float32(PIL.Image.open('/home/behnam/Downloads/fractal.jpg')) according to your setup.

 # imports and basic notebook setup
from cStringIO import StringIO
import numpy as np
import scipy.ndimage as nd
import PIL.Image
from IPython.display import clear_output, Image, display
from google.protobuf import text_format


import caffe


# If your GPU supports CUDA and Caffe was built with CUDA support,
# uncomment the following to run Caffe operations on the GPU.
caffe.set_mode_gpu()
caffe.set_device(0) # select GPU device if multiple devices exist


def showarray(a, fmt='jpeg'):
    a = np.uint8(np.clip(a, 0, 255))
    f = StringIO()
    PIL.Image.fromarray(a).save(f, fmt)
    display(Image(data=f.getvalue()))    
    
#model_path = '../caffe/models/bvlc_googlenet/' # substitute your path here    
model_path = '/home/behnam/.softwares/caffe/models/bvlc_googlenet/' # substitute your path here
net_fn   = model_path + 'deploy.prototxt'
param_fn = model_path + 'bvlc_googlenet.caffemodel'

# Patching model to be able to compute gradients.
# Note that you can also manually add "force_backward: true" line to "deploy.prototxt".
model = caffe.io.caffe_pb2.NetParameter()
text_format.Merge(open(net_fn).read(), model)
model.force_backward = True
open('tmp.prototxt', 'w').write(str(model))

net = caffe.Classifier('tmp.prototxt', param_fn,
                       mean = np.float32([104.0, 116.0, 122.0]), # ImageNet mean, training set dependent
                       channel_swap = (2,1,0)) # the reference model has channels in BGR order instead of RGB

# a couple of utility functions for converting to and from Caffe's input image layout
def preprocess(net, img):
    return np.float32(np.rollaxis(img, 2)[::-1]) - net.transformer.mean['data']
def deprocess(net, img):
    return np.dstack((img + net.transformer.mean['data'])[::-1])


def objective_L2(dst):
    dst.diff[:] = dst.data 

def make_step(net, step_size=1.5, end='inception_4c/output', 
              jitter=32, clip=True, objective=objective_L2):
    '''Basic gradient ascent step.'''

    src = net.blobs['data'] # input image is stored in Net's 'data' blob
    dst = net.blobs[end]

    ox, oy = np.random.randint(-jitter, jitter+1, 2)
    src.data[0] = np.roll(np.roll(src.data[0], ox, -1), oy, -2) # apply jitter shift
            
    net.forward(end=end)
    objective(dst)  # specify the optimization objective
    net.backward(start=end)
    g = src.diff[0]
    # apply normalized ascent step to the input image
    src.data[:] += step_size/np.abs(g).mean() * g

    src.data[0] = np.roll(np.roll(src.data[0], -ox, -1), -oy, -2) # unshift image
            
    if clip:
        bias = net.transformer.mean['data']
        src.data[:] = np.clip(src.data, -bias, 255-bias)
        
        
def deepdream(net, base_img, iter_n=10, octave_n=4, octave_scale=1.4, 
              end='inception_4c/output', clip=True, **step_params):
    # prepare base images for all octaves
    octaves = [preprocess(net, base_img)]
    for i in xrange(octave_n-1):
        octaves.append(nd.zoom(octaves[-1], (1, 1.0/octave_scale,1.0/octave_scale), order=1))
    
    src = net.blobs['data']
    detail = np.zeros_like(octaves[-1]) # allocate image for network-produced details
    for octave, octave_base in enumerate(octaves[::-1]):
        h, w = octave_base.shape[-2:]
        if octave > 0:
            # upscale details from the previous octave
            h1, w1 = detail.shape[-2:]
            detail = nd.zoom(detail, (1, 1.0*h/h1,1.0*w/w1), order=1)

        src.reshape(1,3,h,w) # resize the network's input image size
        src.data[0] = octave_base+detail
        for i in xrange(iter_n):
            make_step(net, end=end, clip=clip, **step_params)
            
            # visualization
            vis = deprocess(net, src.data[0])
            if not clip: # adjust image contrast if clipping is disabled
                vis = vis*(255.0/np.percentile(vis, 99.98))
            showarray(vis)
            print octave, i, end, vis.shape
            clear_output(wait=True)
            
        # extract details produced on the current octave
        detail = src.data[0]-octave_base
    # returning the resulting image
    return deprocess(net, src.data[0])

img = np.float32(PIL.Image.open('/home/behnam/Downloads/fractal.jpg'))
#img = np.float32(PIL.Image.open('/home/behnam/Downloads/sky1024px.jpg'))
showarray(img)

!mkdir frames
frame = img
frame_i = 0
h, w = frame.shape[:2]
s = 0.05 # scale coefficient
for i in xrange(100):
    frame = deepdream(net, frame)
    PIL.Image.fromarray(np.uint8(frame)).save("frames/%04d.jpg"%frame_i)
    frame = nd.affine_transform(frame, [1-s,1-s,1], [h*s/2,w*s/2,0], order=1)
    frame_i += 1

Image(filename='frames/0029.jpg')

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

# imports and basic notebook setup

from cStringIO import StringIO

import numpy as np

import scipy.ndimage as nd

import PIL.Image

from IPython.display import clear_output, Image, display

from google.protobuf import text_format

import caffe

# If your GPU supports CUDA and Caffe was built with CUDA support,

# uncomment the following to run Caffe operations on the GPU.

caffe.set_mode_gpu()

caffe.set_device(0) # select GPU device if multiple devices exist

def showarray(a, fmt='jpeg'):

a = np.uint8(np.clip(a, 0, 255))

f = StringIO()

PIL.Image.fromarray(a).save(f, fmt)

display(Image(data=f.getvalue()))

#model_path = '../caffe/models/bvlc_googlenet/' # substitute your path here

model_path = '/home/behnam/.softwares/caffe/models/bvlc_googlenet/' # substitute your path here

net_fn = model_path + 'deploy.prototxt'

param_fn = model_path + 'bvlc_googlenet.caffemodel'

# Patching model to be able to compute gradients.

# Note that you can also manually add "force_backward: true" line to "deploy.prototxt".

model = caffe.io.caffe_pb2.NetParameter()

text_format.Merge(open(net_fn).read(), model)

model.force_backward = True

open('tmp.prototxt', 'w').write(str(model))

net = caffe.Classifier('tmp.prototxt', param_fn,

mean = np.float32([104.0, 116.0, 122.0]), # ImageNet mean, training set dependent

channel_swap = (2,1,0)) # the reference model has channels in BGR order instead of RGB

# a couple of utility functions for converting to and from Caffe's input image layout

def preprocess(net, img):

return np.float32(np.rollaxis(img, 2)[::-1]) - net.transformer.mean['data']

def deprocess(net, img):

return np.dstack((img + net.transformer.mean['data'])[::-1])

def objective_L2(dst):

dst.diff[:] = dst.data

def make_step(net, step_size=1.5, end='inception_4c/output',

jitter=32, clip=True, objective=objective_L2):

'''Basic gradient ascent step.'''

src = net.blobs['data'] # input image is stored in Net's 'data' blob

dst = net.blobs[end]

ox, oy = np.random.randint(-jitter, jitter+1, 2)

src.data[0] = np.roll(np.roll(src.data[0], ox, -1), oy, -2) # apply jitter shift

net.forward(end=end)

objective(dst) # specify the optimization objective

net.backward(start=end)

g = src.diff[0]

# apply normalized ascent step to the input image

src.data[:] += step_size/np.abs(g).mean() * g

src.data[0] = np.roll(np.roll(src.data[0], -ox, -1), -oy, -2) # unshift image

if clip:

bias = net.transformer.mean['data']

src.data[:] = np.clip(src.data, -bias, 255-bias)

def deepdream(net, base_img, iter_n=10, octave_n=4, octave_scale=1.4,

end='inception_4c/output', clip=True, **step_params):

# prepare base images for all octaves

octaves = [preprocess(net, base_img)]

for i in xrange(octave_n-1):

octaves.append(nd.zoom(octaves[-1], (1, 1.0/octave_scale,1.0/octave_scale), order=1))

src = net.blobs['data']

detail = np.zeros_like(octaves[-1]) # allocate image for network-produced details

for octave, octave_base in enumerate(octaves[::-1]):

h, w = octave_base.shape[-2:]

if octave > 0:

# upscale details from the previous octave

h1, w1 = detail.shape[-2:]

detail = nd.zoom(detail, (1, 1.0*h/h1,1.0*w/w1), order=1)

src.reshape(1,3,h,w) # resize the network's input image size

src.data[0] = octave_base+detail

for i in xrange(iter_n):

make_step(net, end=end, clip=clip, **step_params)

# visualization

vis = deprocess(net, src.data[0])

if not clip: # adjust image contrast if clipping is disabled

vis = vis*(255.0/np.percentile(vis, 99.98))

showarray(vis)

print octave, i, end, vis.shape

clear_output(wait=True)

# extract details produced on the current octave

detail = src.data[0]-octave_base

# returning the resulting image

return deprocess(net, src.data[0])

img = np.float32(PIL.Image.open('/home/behnam/Downloads/fractal.jpg'))

#img = np.float32(PIL.Image.open('/home/behnam/Downloads/sky1024px.jpg'))

showarray(img)

!mkdir frames

frame = img

frame_i = 0

h, w = frame.shape[:2]

s = 0.05 # scale coefficient

for i in xrange(100):

frame = deepdream(net, frame)

PIL.Image.fromarray(np.uint8(frame)).save("frames/%04d.jpg"%frame_i)

frame = nd.affine_transform(frame, [1-s,1-s,1], [h*s/2,w*s/2,0], order=1)

frame_i += 1

Image(filename='frames/0029.jpg')

Car Detection Using Single Shot MultiBox Detector (SSD Convolutional Neural Network) in ROS Using Caffe

Leave a reply

This work is similar to the previous work here, but this time I used Single Shot MultiBox Detector (SSD) for car detection. Installation is similar, clone the SSD Caffe:

git clone https://github.com/weiliu89/caffe.git ssdcaffe
cd ssdcaffe
git checkout 5365d0dccacd18e65f10e840eab28eb65ce0cda

git clone https://github.com/weiliu89/caffe.git ssdcaffe

cd ssdcaffe

git checkout 5365d0dccacd18e65f10e840eab28eb65ce0cda

add the following lines to your Makefile.config

INCLUDE_DIRS += /usr/include/hdf5/serial
LIBRARY_DIRS += /usr/lib/x86_64-linux-gnu/hdf5/serial

1 2	INCLUDE_DIRS += /usr/include/hdf5/serial LIBRARY_DIRS += /usr/lib/x86_64-linux-gnu/hdf5/serial

and build it:

make && make distribute

1	make && make distribute

used video_stream_opencv to stream your video:

<!--
Example of run:
roslaunch video_file.launch video_file:=/home/behnam/Desktop/dash_cam.mp4
-->
<launch>
	<!-- launch video stream -->
		<!--remaping the topic -->
		<remap from="/videofile/image_raw" to="/image_raw"/>
		<include file="$(find video_stream_opencv)/launch/camera.launch" >
	   		<!-- node name and ros graph name -->
		  	<arg name="camera_name" value="videofile" />
		  	<!-- full path to the video file -->
		  	<arg name="video_stream_provider" value="$(arg video_file)" />
		  	<!-- throttling the querying of frames to -->
		  	<arg name="fps" value="30" />
		  	<!-- setting frame_id -->
		  	<arg name="frame_id" value="videofile_frame" />
	</include>
</launch>

<!--

Example of run:

roslaunch video_file.launch video_file:=/home/behnam/Desktop/dash_cam.mp4

-->

</include>

</launch>

download the trained model from here and put them in the model directory.

source /opt/ros/kinetic/setup.bash 
source /home/<user_name>/Autoware/ros/devel/setup.sh 
export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH
roslaunch cv_tracker ssd.launch

source /opt/ros/kinetic/setup.bash

source /home/<user_name>/Autoware/ros/devel/setup.sh

export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH

roslaunch cv_tracker ssd.launch

In my ssd.launch, I have changed my trained network into:

<!-- 
<arg name="network_definition_file" default="$(env HOME)/ssdcaffe/models/VGGNet/VOC0712/SSD_500x500/deploy.prototxt"/>
<arg name="pretrained_model_file" default="$(env HOME)/ssdcaffe/models/VGGNet/VOC0712/SSD_500x500/VGG_VOC0712_SSD_500x500_iter_60000.caffemodel"/> 
-->

<arg name="network_definition_file" default="$(env HOME)/ssdcaffe/models/VGGNet/ilsvrc15/SSD_500x500/deploy.prototxt"/>
<arg name="pretrained_model_file" default="$(env HOME)/ssdcaffe/models/VGGNet/ilsvrc15/SSD_500x500/VGG_ilsvrc15_SSD_500x500_iter_480000.caffemodel"/>

<!--

-->

Now run the following to open rviz:

source /opt/ros/kinetic/setup.bash
source /home/<user_name>/Autoware/ros/devel/setup.sh
export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH
rosrun rviz rviz

source /opt/ros/kinetic/setup.bash

source /home/<user_name>/Autoware/ros/devel/setup.sh

export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH

rosrun rviz rviz

in the rviz, go to add a panel, and add integrated viewer>ImageViewrPlugin.

Now correct the topic in the added panel and you should see detected cars:

Car Detection Using Fast Region-based Convolutional Networks (R-CNN) in ROS with Caffe

1 Reply

To run this, you need to install Fast-RCNN and Autoware. Just in case you got error regarding hd5f when making Fast-RCNN, add the following lines to your Makefile.config

INCLUDE_DIRS += /usr/include/hdf5/serial
LIBRARY_DIRS += /usr/lib/x86_64-linux-gnu/hdf5/serial

1 2	INCLUDE_DIRS += /usr/include/hdf5/serial LIBRARY_DIRS += /usr/lib/x86_64-linux-gnu/hdf5/serial

Now run the following command to start:

source /opt/ros/kinetic/setup.bash
source /home/<user_name>/Autoware/ros/devel/setup.sh
export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH
roslaunch cv_tracker rcnn.launch

source /opt/ros/kinetic/setup.bash

source /home/<user_name>/Autoware/ros/devel/setup.sh

export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH

roslaunch cv_tracker rcnn.launch

if you got an error like :

Check failed: error == cudaSuccess (30 vs. 0) unknown error

1	Check failed: error == cudaSuccess (30 vs. 0) unknown error

That means your graphics card is not ready or accessible, in my everytime I suspend my notebook I get that error and I need a restart :/

now you should publish your video stream on the topic "image_raw", for that purpose I used video_stream_opencv. Here is my launch file:

<!--
Example of run:
roslaunch video_file.launch video_file:=/home/behnam/Desktop/dash_cam.mp4
-->
<launch>
	<!-- launch video stream -->
		<!--remaping the topic -->
		<remap from="/videofile/image_raw" to="/image_raw"/>
		<include file="$(find video_stream_opencv)/launch/camera.launch" >
	   		<!-- node name and ros graph name -->
		  	<arg name="camera_name" value="videofile" />
		  	<!-- full path to the video file -->
		  	<arg name="video_stream_provider" value="$(arg video_file)" />
		  	<!-- throttling the querying of frames to -->
		  	<arg name="fps" value="30" />
		  	<!-- setting frame_id -->
		  	<arg name="frame_id" value="videofile_frame" />
	</include>
</launch>

<!--

Example of run:

roslaunch video_file.launch video_file:=/home/behnam/Desktop/dash_cam.mp4

-->

</include>

</launch>

Now run the following to open rviz:

source /opt/ros/kinetic/setup.bash
source /home/<user_name>/Autoware/ros/devel/setup.sh
export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH
rosrun rviz rviz

source /opt/ros/kinetic/setup.bash

source /home/<user_name>/Autoware/ros/devel/setup.sh

export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH

rosrun rviz rviz

in the rviz, go to add a panel, and add integrated viewer>ImageViewrPlugin.

Now correct the topic in the added panel and you should see detected cars: