4.Deep Dream 模型

1.技术原理

以某一个通道的平均值作为优化目标，不断让神经网络去调整输入图像的像素值，让输出的图片中逐渐出现该通道"喜欢"的纹理和图案

2.TensorFlow中的Deep Dream模型实践

导入Inception:

下载文件后复制到chapter4目录下：

https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip

# coding:utf-8 from __future__ import print_function import numpy as np import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph = tf.Graph() sess = tf.InteractiveSession(graph=graph) model_fn = 'tensorflow_inception_graph.pb' with tf.gfile.GFile(model_fn, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) t_input = tf.placeholder(np.float32, name='input') imagenet_mean = 117.0 t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {'input': t_preprocessed}) #找到卷积层 layers = [op.name for op in graph.get_operations() if op.type == 'Conv2D' and 'import/' in op.name] #输出卷积层 print('Number of layers', len(layers)) #输出mixed4d_3x3_bottleneck_pre_relu的形状 name = 'mixed4d_3x3_bottleneck_pre_relu' print('shape of %s: %s' % (name, str(graph.get_tensor_by_name('import/' + name + ':0').get_shape()))) #t_input:设置一个占位符，在后面程序把图像数据传递给t_input #tf.expand_dims:在原始（height,width,channel）输入前增加一堆（batch,height,width,channel） #减去均值，在训练时Inception模型已经做了减去均值的预处理，此处保持同样预处理才能输入一致 #导入模型：tf.import_graph_def(graph_def, {'input': t_preprocessed})

运行结果:

生成原始的Deep Dream图像

# coding: utf-8 from __future__ import print_function import os from io import BytesIO import numpy as np from functools import partial import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph = tf.Graph() model_fn = 'tensorflow_inception_graph.pb' sess = tf.InteractiveSession(graph=graph) with tf.gfile.GFile(model_fn, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) t_input = tf.placeholder(np.float32, name='input') imagenet_mean = 117.0 t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {'input': t_preprocessed}) #定义一个保存图片的函数，把numpy.ndarray保存成文件的形式 def savearray(img_array, img_name): # 用 PIL 替代 scipy.misc.toimage img = Image.fromarray(np.uint8(img_array)) img.save(img_name) print('img saved: %s' % img_name) def render_naive(t_obj, img0, iter_n=20, step=1.0): # t_score是优化目标。它是t_obj的平均值 # 结合调用处看，实际上就是layer_output[:, :, :, channel]的平均值 t_score = tf.reduce_mean(t_obj) # 计算t_score对t_input的梯度 t_grad = tf.gradients(t_score, t_input)[0] # 创建新图 img = img0.copy() for i in range(iter_n): # 在sess中计算梯度，以及当前的score g, score = sess.run([t_grad, t_score], {t_input: img}) # 对img应用梯度。step可以看做"学习率" g /= g.std() + 1e-8 img += g * step print('score(mean)=%f' % (score)) # 保存图片 savearray(img, 'naive.jpg') # 定义卷积层、通道数，并取出对应的tensor name = 'mixed4d_3x3_bottleneck_pre_relu' channel = 139 layer_output = graph.get_tensor_by_name("import/%s:0" % name) # 定义原始的图像噪声 img_noise = np.random.uniform(size=(224, 224, 3)) + 100.0 # 调用render_naive函数渲染 render_naive(layer_output[:, :, :, channel], img_noise, iter_n=20) #t_obj卷积层某个通道的值，t_score是t_obj的平均值

运行结果：

生成更大尺寸的Deep Dream图像

不是对整张图片做优化，把图片分成多个部分，每次对一个部分做优化，达到优化时只消耗固定大小的内存。

# coding:utf-8 from __future__ import print_function import os from io import BytesIO import numpy as np from functools import partial import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph = tf.Graph() model_fn = 'tensorflow_inception_graph.pb' sess = tf.InteractiveSession(graph=graph) with tf.gfile.GFile(model_fn, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) t_input = tf.placeholder(np.float32, name='input') imagenet_mean = 117.0 t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {'input': t_preprocessed}) def savearray(img_array, img_name): img = Image.fromarray(np.uint8(np.clip(img_array, 0, 255))) img.save(img_name) print('img saved: %s' % img_name) #放大rario倍，先确定原先像素的范围，计算img的最值，用scipy.misc.imresize后，再将像素缩放回去 def resize_ratio(img, ratio): min_val = img.min() max_val = img.max() img = (img - min_val) / (max_val - min_val) * 255 # 用 PIL 替代 scipy.misc.imresize h, w = img.shape[:2] new_h, new_w = int(h * ratio), int(w * ratio) img = Image.fromarray(np.uint8(np.clip(img, 0, 255))) img = img.resize((new_w, new_h), Image.LANCZOS) img = np.float32(img) img = img / 255 * (max_val - min_val) + min_val return img #对任意大小的图像计算梯度 def calc_grad_tiled(img, t_grad, tile_size=512): sz = tile_size h, w = img.shape[:2] sx, sy = np.random.randint(sz, size=2) #整体移动，在图像边缘的像素就会被移动到图像中间，避免边缘效应 img_shift = np.roll(np.roll(img, sx, 1), sy, 0) grad = np.zeros_like(img) for y in range(0, max(h - sz // 2, sz), sz): for x in range(0, max(w - sz // 2, sz), sz): sub = img_shift[y:y + sz, x:x + sz] g = sess.run(t_grad, {t_input: sub}) grad[y:y + sz, x:x + sz] = g return np.roll(np.roll(grad, -sx, 1), -sy, 0) #生成最大尺寸图像的函数，先生成小尺寸的图像，调用resize_ratio放大octave_scale倍，再进行计算，进行octave_n-1次 #octave_n决定图像大小 def render_multiscale(t_obj, img0, iter_n=10, step=1.0, octave_n=3, octave_scale=1.4): t_score = tf.reduce_mean(t_obj) t_grad = tf.gradients(t_score, t_input)[0] img = img0.copy() for octave in range(octave_n): if octave > 0: img = resize_ratio(img, octave_scale) for i in range(iter_n): g = calc_grad_tiled(img, t_grad) g /= g.std() + 1e-8 img += g * step print('.', end=' ') savearray(img, 'multiscale.jpg') if __name__ == '__main__': name = 'mixed4d_3x3_bottleneck_pre_relu' channel = 139 img_noise = np.random.uniform(size=(224, 224, 3)) + 100.0 layer_output = graph.get_tensor_by_name("import/%s:0" % name) render_multiscale(layer_output[:, :, :, channel], img_noise, iter_n=20)

运行结果：

生成更高质量的Deep Dream图像

高频：图像灰度、颜色、明度变化比较强烈

低频：图像变化不大的地方，风格、色块，图片更柔和

将梯度做分解（拉普拉斯金字塔），分为高频和低频梯度，再放大低频梯度，图片更柔和

# coding:utf-8 from __future__ import print_function import os from io import BytesIO import numpy as np from functools import partial import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph = tf.Graph() model_fn = 'tensorflow_inception_graph.pb' sess = tf.InteractiveSession(graph=graph) with tf.gfile.GFile(model_fn, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) t_input = tf.placeholder(np.float32, name='input') imagenet_mean = 117.0 t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {'input': t_preprocessed}) def savearray(img_array, img_name): img = Image.fromarray(np.uint8(np.clip(img_array, 0, 255))) img.save(img_name) print('img saved: %s' % img_name) def resize_ratio(img, ratio): min_val = img.min() max_val = img.max() img = (img - min_val) / (max_val - min_val) * 255 h, w = img.shape[:2] new_h, new_w = int(h * ratio), int(w * ratio) img = Image.fromarray(np.uint8(np.clip(img, 0, 255))) img = img.resize((new_w, new_h), Image.LANCZOS) img = np.float32(img) img = img / 255 * (max_val - min_val) + min_val return img def calc_grad_tiled(img, t_grad, tile_size=512): sz = tile_size h, w = img.shape[:2] sx, sy = np.random.randint(sz, size=2) img_shift = np.roll(np.roll(img, sx, 1), sy, 0) grad = np.zeros_like(img) for y in range(0, max(h - sz // 2, sz), sz): for x in range(0, max(w - sz // 2, sz), sz): sub = img_shift[y:y + sz, x:x + sz] g = sess.run(t_grad, {t_input: sub}) grad[y:y + sz, x:x + sz] = g return np.roll(np.roll(grad, -sx, 1), -sy, 0) k = np.float32([1, 4, 6, 4, 1]) k = np.outer(k, k) k5x5 = k[:, :, None, None] / k.sum() * np.eye(3, dtype=np.float32) #将图像分为高频和低频成分 def lap_split(img): with tf.name_scope('split'): lo = tf.nn.conv2d(img, k5x5, [1, 2, 2, 1], 'SAME') lo2 = tf.nn.conv2d_transpose(lo, k5x5 * 4, tf.shape(img), [1, 2, 2, 1]) hi = img - lo2 return lo, hi #分为n层金字塔 def lap_split_n(img, n): levels = [] for i in range(n): #分为高低频，高频保存到levels中，低频继续分解 img, hi = lap_split(img) levels.append(hi) levels.append(img) return levels[::-1] #将拉普拉斯金字塔还原到原始图像 def lap_merge(levels): img = levels[0] for hi in levels[1:]: with tf.name_scope('merge'): img = tf.nn.conv2d_transpose(img, k5x5 * 4, tf.shape(hi), [1, 2, 2, 1]) + hi return img #标准化 def normalize_std(img, eps=1e-10): with tf.name_scope('normalize'): std = tf.sqrt(tf.reduce_mean(tf.square(img))) return img / tf.maximum(std, eps) #拉普拉斯金字塔标准化 def lap_normalize(img, scale_n=4): img = tf.expand_dims(img, 0) tlevels = lap_split_n(img, scale_n) tlevels = list(map(normalize_std, tlevels)) out = lap_merge(tlevels) return out[0, :, :, :] #将一个对Tensor定义的函数转换成一个正常的对numpy.ndarray定义的函数 def tffunc(*argtypes): placeholders = list(map(tf.placeholder, argtypes)) def wrap(f): out = f(*placeholders) def wrapper(*args, **kw): return out.eval(dict(zip(placeholders, args)), session=kw.get('session')) return wrapper return wrap #生成图像 def render_lapnorm(t_obj, img0, iter_n=10, step=1.0, octave_n=3, octave_scale=1.4, lap_n=4): t_score = tf.reduce_mean(t_obj) t_grad = tf.gradients(t_score, t_input)[0] lap_norm_func = tffunc(np.float32)(partial(lap_normalize, scale_n=lap_n)) img = img0.copy() for octave in range(octave_n): if octave > 0: img = resize_ratio(img, octave_scale) for i in range(iter_n): g = calc_grad_tiled(img, t_grad) g = lap_norm_func(g) img += g * step print('.', end=' ') savearray(img, 'lapnorm.jpg') if __name__ == '__main__': name = 'mixed4d_3x3_bottleneck_pre_relu' channel = 139 img_noise = np.random.uniform(size=(224, 224, 3)) + 100.0 layer_output = graph.get_tensor_by_name("import/%s:0" % name) render_lapnorm(layer_output[:, :, :, channel], img_noise, iter_n=20)

运行结束：

最终的Deep Dream 模型

添加背景

# coding:utf-8 from __future__ import print_function import os from io import BytesIO import numpy as np from functools import partial import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph = tf.Graph() model_fn = 'tensorflow_inception_graph.pb' sess = tf.InteractiveSession(graph=graph) with tf.gfile.GFile(model_fn, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) t_input = tf.placeholder(np.float32, name='input') imagenet_mean = 117.0 t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {'input': t_preprocessed}) def savearray(img_array, img_name): img = Image.fromarray(np.uint8(np.clip(img_array, 0, 255))) img.save(img_name) print('img saved: %s' % img_name) def visstd(a, s=0.1): return (a - a.mean()) / max(a.std(), 1e-4) * s + 0.5 def resize_ratio(img, ratio): min_val = img.min() max_val = img.max() img = (img - min_val) / (max_val - min_val) * 255 h, w = img.shape[:2] new_h, new_w = int(h * ratio), int(w * ratio) img = Image.fromarray(np.uint8(np.clip(img, 0, 255))) img = img.resize((new_w, new_h), Image.LANCZOS) img = np.float32(img) img = img / 255 * (max_val - min_val) + min_val return img #缩小原图像 def resize(img, hw): min_val = img.min() max_val = img.max() img = (img - min_val) / (max_val - min_val) * 255 img = Image.fromarray(np.uint8(np.clip(img, 0, 255))) img = img.resize((hw[1], hw[0]), Image.LANCZOS) img = np.float32(img) img = img / 255 * (max_val - min_val) + min_val return img def calc_grad_tiled(img, t_grad, tile_size=512): sz = tile_size h, w = img.shape[:2] sx, sy = np.random.randint(sz, size=2) img_shift = np.roll(np.roll(img, sx, 1), sy, 0) grad = np.zeros_like(img) for y in range(0, max(h - sz // 2, sz), sz): for x in range(0, max(w - sz // 2, sz), sz): sub = img_shift[y:y + sz, x:x + sz] g = sess.run(t_grad, {t_input: sub}) grad[y:y + sz, x:x + sz] = g return np.roll(np.roll(grad, -sx, 1), -sy, 0) def tffunc(*argtypes): placeholders = list(map(tf.placeholder, argtypes)) def wrap(f): out = f(*placeholders) def wrapper(*args, **kw): return out.eval(dict(zip(placeholders, args)), session=kw.get('session')) return wrapper return wrap #为了保证图像的质量，进行高低频分解，直接缩小原图像 def render_deepdream(t_obj, img0, iter_n=10, step=1.5, octave_n=4, octave_scale=1.4): t_score = tf.reduce_mean(t_obj) t_grad = tf.gradients(t_score, t_input)[0] img = img0 octaves = [] for i in range(octave_n - 1): hw = img.shape[:2] lo = resize(img, np.int32(np.float32(hw) / octave_scale)) hi = img - resize(lo, hw) img = lo octaves.append(hi) for octave in range(octave_n): if octave > 0: hi = octaves[-octave] img = resize(img, hi.shape[:2]) + hi for i in range(iter_n): g = calc_grad_tiled(img, t_grad) img += g * (step / (np.abs(g).mean() + 1e-7)) print('.', end=' ') img = img.clip(0, 255) savearray(img, 'deepdream.jpg') if __name__ == '__main__': img0 = PIL.Image.open('test.jpg') img0 = np.float32(img0) name = 'mixed4d_3x3_bottleneck_pre_relu' channel = 139 layer_output = graph.get_tensor_by_name("import/%s:0" % name) render_deepdream(layer_output[:, :, :, channel], img0)

运行结果：

带有动物的DeepDream图片

# coding:utf-8 from __future__ import print_function import numpy as np import PIL.Image from PIL import Image import tensorflow.compat.v1 as tf tf.disable_v2_behavior() graph = tf.Graph() model_fn = 'tensorflow_inception_graph.pb' sess = tf.InteractiveSession(graph=graph) with tf.gfile.GFile(model_fn, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) t_input = tf.placeholder(np.float32, name='input') imagenet_mean = 117.0 t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {'input': t_preprocessed}) def savearray(img_array, img_name): img = Image.fromarray(np.uint8(np.clip(img_array, 0, 255))) img.save(img_name) print('img saved: %s' % img_name) def resize(img, hw): min_val = img.min() max_val = img.max() img = (img - min_val) / (max_val - min_val) * 255 img = Image.fromarray(np.uint8(np.clip(img, 0, 255))) img = img.resize((hw[1], hw[0]), Image.LANCZOS) img = np.float32(img) img = img / 255 * (max_val - min_val) + min_val return img def calc_grad_tiled(img, t_grad, tile_size=512): sz = tile_size h, w = img.shape[:2] sx, sy = np.random.randint(sz, size=2) img_shift = np.roll(np.roll(img, sx, 1), sy, 0) grad = np.zeros_like(img) for y in range(0, max(h - sz // 2, sz), sz): for x in range(0, max(w - sz // 2, sz), sz): sub = img_shift[y:y + sz, x:x + sz] g = sess.run(t_grad, {t_input: sub}) grad[y:y + sz, x:x + sz] = g return np.roll(np.roll(grad, -sx, 1), -sy, 0) def render_deepdream(t_obj, img0, iter_n=10, step=1.5, octave_n=4, octave_scale=1.4): t_score = tf.reduce_mean(t_obj) t_grad = tf.gradients(t_score, t_input)[0] img = img0 octaves = [] for i in range(octave_n - 1): hw = img.shape[:2] lo = resize(img, np.int32(np.float32(hw) / octave_scale)) hi = img - resize(lo, hw) img = lo octaves.append(hi) for octave in range(octave_n): if octave > 0: hi = octaves[-octave] img = resize(img, hi.shape[:2]) + hi for i in range(iter_n): g = calc_grad_tiled(img, t_grad) img += g * (step / (np.abs(g).mean() + 1e-7)) print('.', end=' ') print() img = img.clip(0, 255) savearray(img, 'deepdream_animals.jpg') if __name__ == '__main__': # 用一张天空或风景照作为输入，动物图案更容易显现 img0 = PIL.Image.open('test.jpg') img0 = np.float32(img0) # 关键：使用 tf.square 放大 mixed4c 整个层的输出 # mixed4c 包含大量动物特征（眼睛、羽毛、爪子等） name = 'mixed4c' layer_output = graph.get_tensor_by_name("import/%s:0" % name) render_deepdream(tf.square(layer_output), img0, iter_n=20)