Tag Archives: Caffe

Deep Dreams with Caffe on Ubuntu 16.04

Googlenet Model

Download the bvlc_googlenet.caffemodel from https://github.com/BVLC/caffe/tree/master/models/bvlc_googlenet

and put it in
caffe/models/bvlc_googlenet/

PIP

sudo apt install python-pip

1	sudo apt install python-pip

IPython, scipy, Jupyter, protobuf, scikit-image

Always install in the user space with –user

pip install --user IPython==5.0 scipy Jupyter protobuf scikit-image

1	pip install --user IPython==5.0 scipy Jupyter protobuf scikit-image

Running jupyter notebook

export PYTHONPATH=/home/behnam/usr/python:$PYTHONPATH
jupyter notebook

1 2	export PYTHONPATH=/home/behnam/usr/python:$PYTHONPATH jupyter notebook

open a new notebook and paste the following into it and correct the “model_path” and

img = np.float32(PIL.Image.open(‘/home/behnam/Downloads/fractal.jpg’)) according to your setup.

 # imports and basic notebook setup
from cStringIO import StringIO
import numpy as np
import scipy.ndimage as nd
import PIL.Image
from IPython.display import clear_output, Image, display
from google.protobuf import text_format


import caffe


# If your GPU supports CUDA and Caffe was built with CUDA support,
# uncomment the following to run Caffe operations on the GPU.
caffe.set_mode_gpu()
caffe.set_device(0) # select GPU device if multiple devices exist


def showarray(a, fmt='jpeg'):
    a = np.uint8(np.clip(a, 0, 255))
    f = StringIO()
    PIL.Image.fromarray(a).save(f, fmt)
    display(Image(data=f.getvalue()))    
    
#model_path = '../caffe/models/bvlc_googlenet/' # substitute your path here    
model_path = '/home/behnam/.softwares/caffe/models/bvlc_googlenet/' # substitute your path here
net_fn   = model_path + 'deploy.prototxt'
param_fn = model_path + 'bvlc_googlenet.caffemodel'

# Patching model to be able to compute gradients.
# Note that you can also manually add "force_backward: true" line to "deploy.prototxt".
model = caffe.io.caffe_pb2.NetParameter()
text_format.Merge(open(net_fn).read(), model)
model.force_backward = True
open('tmp.prototxt', 'w').write(str(model))

net = caffe.Classifier('tmp.prototxt', param_fn,
                       mean = np.float32([104.0, 116.0, 122.0]), # ImageNet mean, training set dependent
                       channel_swap = (2,1,0)) # the reference model has channels in BGR order instead of RGB

# a couple of utility functions for converting to and from Caffe's input image layout
def preprocess(net, img):
    return np.float32(np.rollaxis(img, 2)[::-1]) - net.transformer.mean['data']
def deprocess(net, img):
    return np.dstack((img + net.transformer.mean['data'])[::-1])


def objective_L2(dst):
    dst.diff[:] = dst.data 

def make_step(net, step_size=1.5, end='inception_4c/output', 
              jitter=32, clip=True, objective=objective_L2):
    '''Basic gradient ascent step.'''

    src = net.blobs['data'] # input image is stored in Net's 'data' blob
    dst = net.blobs[end]

    ox, oy = np.random.randint(-jitter, jitter+1, 2)
    src.data[0] = np.roll(np.roll(src.data[0], ox, -1), oy, -2) # apply jitter shift
            
    net.forward(end=end)
    objective(dst)  # specify the optimization objective
    net.backward(start=end)
    g = src.diff[0]
    # apply normalized ascent step to the input image
    src.data[:] += step_size/np.abs(g).mean() * g

    src.data[0] = np.roll(np.roll(src.data[0], -ox, -1), -oy, -2) # unshift image
            
    if clip:
        bias = net.transformer.mean['data']
        src.data[:] = np.clip(src.data, -bias, 255-bias)
        
        
def deepdream(net, base_img, iter_n=10, octave_n=4, octave_scale=1.4, 
              end='inception_4c/output', clip=True, **step_params):
    # prepare base images for all octaves
    octaves = [preprocess(net, base_img)]
    for i in xrange(octave_n-1):
        octaves.append(nd.zoom(octaves[-1], (1, 1.0/octave_scale,1.0/octave_scale), order=1))
    
    src = net.blobs['data']
    detail = np.zeros_like(octaves[-1]) # allocate image for network-produced details
    for octave, octave_base in enumerate(octaves[::-1]):
        h, w = octave_base.shape[-2:]
        if octave > 0:
            # upscale details from the previous octave
            h1, w1 = detail.shape[-2:]
            detail = nd.zoom(detail, (1, 1.0*h/h1,1.0*w/w1), order=1)

        src.reshape(1,3,h,w) # resize the network's input image size
        src.data[0] = octave_base+detail
        for i in xrange(iter_n):
            make_step(net, end=end, clip=clip, **step_params)
            
            # visualization
            vis = deprocess(net, src.data[0])
            if not clip: # adjust image contrast if clipping is disabled
                vis = vis*(255.0/np.percentile(vis, 99.98))
            showarray(vis)
            print octave, i, end, vis.shape
            clear_output(wait=True)
            
        # extract details produced on the current octave
        detail = src.data[0]-octave_base
    # returning the resulting image
    return deprocess(net, src.data[0])

img = np.float32(PIL.Image.open('/home/behnam/Downloads/fractal.jpg'))
#img = np.float32(PIL.Image.open('/home/behnam/Downloads/sky1024px.jpg'))
showarray(img)

!mkdir frames
frame = img
frame_i = 0
h, w = frame.shape[:2]
s = 0.05 # scale coefficient
for i in xrange(100):
    frame = deepdream(net, frame)
    PIL.Image.fromarray(np.uint8(frame)).save("frames/%04d.jpg"%frame_i)
    frame = nd.affine_transform(frame, [1-s,1-s,1], [h*s/2,w*s/2,0], order=1)
    frame_i += 1

Image(filename='frames/0029.jpg')

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

# imports and basic notebook setup

from cStringIO import StringIO

import numpy as np

import scipy.ndimage as nd

import PIL.Image

from IPython.display import clear_output, Image, display

from google.protobuf import text_format

import caffe

# If your GPU supports CUDA and Caffe was built with CUDA support,

# uncomment the following to run Caffe operations on the GPU.

caffe.set_mode_gpu()

caffe.set_device(0) # select GPU device if multiple devices exist

def showarray(a, fmt='jpeg'):

a = np.uint8(np.clip(a, 0, 255))

f = StringIO()

PIL.Image.fromarray(a).save(f, fmt)

display(Image(data=f.getvalue()))

#model_path = '../caffe/models/bvlc_googlenet/' # substitute your path here

model_path = '/home/behnam/.softwares/caffe/models/bvlc_googlenet/' # substitute your path here

net_fn = model_path + 'deploy.prototxt'

param_fn = model_path + 'bvlc_googlenet.caffemodel'

# Patching model to be able to compute gradients.

# Note that you can also manually add "force_backward: true" line to "deploy.prototxt".

model = caffe.io.caffe_pb2.NetParameter()

text_format.Merge(open(net_fn).read(), model)

model.force_backward = True

open('tmp.prototxt', 'w').write(str(model))

net = caffe.Classifier('tmp.prototxt', param_fn,

mean = np.float32([104.0, 116.0, 122.0]), # ImageNet mean, training set dependent

channel_swap = (2,1,0)) # the reference model has channels in BGR order instead of RGB

# a couple of utility functions for converting to and from Caffe's input image layout

def preprocess(net, img):

return np.float32(np.rollaxis(img, 2)[::-1]) - net.transformer.mean['data']

def deprocess(net, img):

return np.dstack((img + net.transformer.mean['data'])[::-1])

def objective_L2(dst):

dst.diff[:] = dst.data

def make_step(net, step_size=1.5, end='inception_4c/output',

jitter=32, clip=True, objective=objective_L2):

'''Basic gradient ascent step.'''

src = net.blobs['data'] # input image is stored in Net's 'data' blob

dst = net.blobs[end]

ox, oy = np.random.randint(-jitter, jitter+1, 2)

src.data[0] = np.roll(np.roll(src.data[0], ox, -1), oy, -2) # apply jitter shift

net.forward(end=end)

objective(dst) # specify the optimization objective

net.backward(start=end)

g = src.diff[0]

# apply normalized ascent step to the input image

src.data[:] += step_size/np.abs(g).mean() * g

src.data[0] = np.roll(np.roll(src.data[0], -ox, -1), -oy, -2) # unshift image

if clip:

bias = net.transformer.mean['data']

src.data[:] = np.clip(src.data, -bias, 255-bias)

def deepdream(net, base_img, iter_n=10, octave_n=4, octave_scale=1.4,

end='inception_4c/output', clip=True, **step_params):

# prepare base images for all octaves

octaves = [preprocess(net, base_img)]

for i in xrange(octave_n-1):

octaves.append(nd.zoom(octaves[-1], (1, 1.0/octave_scale,1.0/octave_scale), order=1))

src = net.blobs['data']

detail = np.zeros_like(octaves[-1]) # allocate image for network-produced details

for octave, octave_base in enumerate(octaves[::-1]):

h, w = octave_base.shape[-2:]

if octave > 0:

# upscale details from the previous octave

h1, w1 = detail.shape[-2:]

detail = nd.zoom(detail, (1, 1.0*h/h1,1.0*w/w1), order=1)

src.reshape(1,3,h,w) # resize the network's input image size

src.data[0] = octave_base+detail

for i in xrange(iter_n):

make_step(net, end=end, clip=clip, **step_params)

# visualization

vis = deprocess(net, src.data[0])

if not clip: # adjust image contrast if clipping is disabled

vis = vis*(255.0/np.percentile(vis, 99.98))

showarray(vis)

print octave, i, end, vis.shape

clear_output(wait=True)

# extract details produced on the current octave

detail = src.data[0]-octave_base

# returning the resulting image

return deprocess(net, src.data[0])

img = np.float32(PIL.Image.open('/home/behnam/Downloads/fractal.jpg'))

#img = np.float32(PIL.Image.open('/home/behnam/Downloads/sky1024px.jpg'))

showarray(img)

!mkdir frames

frame = img

frame_i = 0

h, w = frame.shape[:2]

s = 0.05 # scale coefficient

for i in xrange(100):

frame = deepdream(net, frame)

PIL.Image.fromarray(np.uint8(frame)).save("frames/%04d.jpg"%frame_i)

frame = nd.affine_transform(frame, [1-s,1-s,1], [h*s/2,w*s/2,0], order=1)

frame_i += 1

Image(filename='frames/0029.jpg')

Car Detection Using Single Shot MultiBox Detector (SSD Convolutional Neural Network) in ROS Using Caffe

Leave a reply

This work is similar to the previous work here, but this time I used Single Shot MultiBox Detector (SSD) for car detection. Installation is similar, clone the SSD Caffe:

git clone https://github.com/weiliu89/caffe.git ssdcaffe
cd ssdcaffe
git checkout 5365d0dccacd18e65f10e840eab28eb65ce0cda

git clone https://github.com/weiliu89/caffe.git ssdcaffe

cd ssdcaffe

git checkout 5365d0dccacd18e65f10e840eab28eb65ce0cda

add the following lines to your Makefile.config

INCLUDE_DIRS += /usr/include/hdf5/serial
LIBRARY_DIRS += /usr/lib/x86_64-linux-gnu/hdf5/serial

1 2	INCLUDE_DIRS += /usr/include/hdf5/serial LIBRARY_DIRS += /usr/lib/x86_64-linux-gnu/hdf5/serial

and build it:

make && make distribute

1	make && make distribute

used video_stream_opencv to stream your video:

<!--
Example of run:
roslaunch video_file.launch video_file:=/home/behnam/Desktop/dash_cam.mp4
-->
<launch>
	<!-- launch video stream -->
		<!--remaping the topic -->
		<remap from="/videofile/image_raw" to="/image_raw"/>
		<include file="$(find video_stream_opencv)/launch/camera.launch" >
	   		<!-- node name and ros graph name -->
		  	<arg name="camera_name" value="videofile" />
		  	<!-- full path to the video file -->
		  	<arg name="video_stream_provider" value="$(arg video_file)" />
		  	<!-- throttling the querying of frames to -->
		  	<arg name="fps" value="30" />
		  	<!-- setting frame_id -->
		  	<arg name="frame_id" value="videofile_frame" />
	</include>
</launch>

<!--

Example of run:

roslaunch video_file.launch video_file:=/home/behnam/Desktop/dash_cam.mp4

-->

</include>

</launch>

download the trained model from here and put them in the model directory.

source /opt/ros/kinetic/setup.bash 
source /home/<user_name>/Autoware/ros/devel/setup.sh 
export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH
roslaunch cv_tracker ssd.launch

source /opt/ros/kinetic/setup.bash

source /home/<user_name>/Autoware/ros/devel/setup.sh

export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH

roslaunch cv_tracker ssd.launch

In my ssd.launch, I have changed my trained network into:

<!-- 
<arg name="network_definition_file" default="$(env HOME)/ssdcaffe/models/VGGNet/VOC0712/SSD_500x500/deploy.prototxt"/>
<arg name="pretrained_model_file" default="$(env HOME)/ssdcaffe/models/VGGNet/VOC0712/SSD_500x500/VGG_VOC0712_SSD_500x500_iter_60000.caffemodel"/> 
-->

<arg name="network_definition_file" default="$(env HOME)/ssdcaffe/models/VGGNet/ilsvrc15/SSD_500x500/deploy.prototxt"/>
<arg name="pretrained_model_file" default="$(env HOME)/ssdcaffe/models/VGGNet/ilsvrc15/SSD_500x500/VGG_ilsvrc15_SSD_500x500_iter_480000.caffemodel"/>

<!--

-->

Now run the following to open rviz:

source /opt/ros/kinetic/setup.bash
source /home/<user_name>/Autoware/ros/devel/setup.sh
export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH
rosrun rviz rviz

source /opt/ros/kinetic/setup.bash

source /home/<user_name>/Autoware/ros/devel/setup.sh

export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH

rosrun rviz rviz

in the rviz, go to add a panel, and add integrated viewer>ImageViewrPlugin.

Now correct the topic in the added panel and you should see detected cars:

Car Detection Using Fast Region-based Convolutional Networks (R-CNN) in ROS with Caffe

1 Reply

To run this, you need to install Fast-RCNN and Autoware. Just in case you got error regarding hd5f when making Fast-RCNN, add the following lines to your Makefile.config

INCLUDE_DIRS += /usr/include/hdf5/serial
LIBRARY_DIRS += /usr/lib/x86_64-linux-gnu/hdf5/serial

1 2	INCLUDE_DIRS += /usr/include/hdf5/serial LIBRARY_DIRS += /usr/lib/x86_64-linux-gnu/hdf5/serial

Now run the following command to start:

source /opt/ros/kinetic/setup.bash
source /home/<user_name>/Autoware/ros/devel/setup.sh
export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH
roslaunch cv_tracker rcnn.launch

source /opt/ros/kinetic/setup.bash

source /home/<user_name>/Autoware/ros/devel/setup.sh

export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH

roslaunch cv_tracker rcnn.launch

if you got an error like :

Check failed: error == cudaSuccess (30 vs. 0) unknown error

1	Check failed: error == cudaSuccess (30 vs. 0) unknown error

That means your graphics card is not ready or accessible, in my everytime I suspend my notebook I get that error and I need a restart :/

now you should publish your video stream on the topic “image_raw”, for that purpose I used video_stream_opencv. Here is my launch file:

<!--
Example of run:
roslaunch video_file.launch video_file:=/home/behnam/Desktop/dash_cam.mp4
-->
<launch>
	<!-- launch video stream -->
		<!--remaping the topic -->
		<remap from="/videofile/image_raw" to="/image_raw"/>
		<include file="$(find video_stream_opencv)/launch/camera.launch" >
	   		<!-- node name and ros graph name -->
		  	<arg name="camera_name" value="videofile" />
		  	<!-- full path to the video file -->
		  	<arg name="video_stream_provider" value="$(arg video_file)" />
		  	<!-- throttling the querying of frames to -->
		  	<arg name="fps" value="30" />
		  	<!-- setting frame_id -->
		  	<arg name="frame_id" value="videofile_frame" />
	</include>
</launch>

<!--

Example of run:

roslaunch video_file.launch video_file:=/home/behnam/Desktop/dash_cam.mp4

-->

</include>

</launch>

Now run the following to open rviz:

source /opt/ros/kinetic/setup.bash
source /home/<user_name>/Autoware/ros/devel/setup.sh
export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH
rosrun rviz rviz

source /opt/ros/kinetic/setup.bash

source /home/<user_name>/Autoware/ros/devel/setup.sh

export ROS_PACKAGE_PATH=/home/<user_name>/Autoware/:$ROS_PACKAGE_PATH

rosrun rviz rviz

in the rviz, go to add a panel, and add integrated viewer>ImageViewrPlugin.

Now correct the topic in the added panel and you should see detected cars:

Cookie	Duration	Description
cookielawinfo-checbox-analytics	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookie is used to store the user consent for the cookies in the category "Analytics".
cookielawinfo-checbox-functional	11 months	The cookie is set by GDPR cookie consent to record the user consent for the cookies in the category "Functional".
cookielawinfo-checbox-others	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookie is used to store the user consent for the cookies in the category "Other.
cookielawinfo-checkbox-necessary	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookies is used to store the user consent for the cookies in the category "Necessary".
cookielawinfo-checkbox-performance	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookie is used to store the user consent for the cookies in the category "Performance".
viewed_cookie_policy	11 months	The cookie is set by the GDPR Cookie Consent plugin and is used to store whether or not user has consented to the use of cookies. It does not store any personal data.