Fork me on GitHub

tf.keras.Model子类注意项

字数统计: 1.2k字   |   阅读时长≈ 6分

tf.keras.Model子类的注意项

自定义网络层

自定义网络层需要继承 tf.keras.layers.Layer，自定义的层的 trainable 参数默认为 True，即默认可以训练， 如以下定义了两个层：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
class MaskMean(tf.keras.layers.Layer):
    def __init__(self):
        super(MaskMean, self).__init__()
        self.dropout = tf.keras.layers.Dropout(0.1)
        # self.trainable_variables.append(self.dropout)

    def call(self, inputs, **kwargs):
        print(type(inputs), len(inputs))
        dropout = self.dropout(inputs[0])
        mean = self.batch_boolean_mask_and_mean(dropout, inputs[1])
        return mean

    def batch_boolean_mask_and_mean(self, tensor, mask):
        """
        mask with batch
        :param tensor: needed to be masked
        :param mask: masker
        :return: [batch_size, embedding_size]
        """
        assert tensor.shape[0] == mask.shape[0]
        # [batch_size, embedding_size]
        mean_embeddings = np.zeros([tensor.shape[0], tensor.shape[2]])
        # example[max_seq_len, embedding_size], masker[max_seq_len,]
        for i, (example, masker) in enumerate(zip(tensor, mask)):
            masked = tf.boolean_mask(example, masker)
            mean_embeddings[i] = tf.reduce_mean(masked, axis=0).numpy()
        return mean_embeddings
1
2
3
4
5
6
7
8
9
10
11
class Predict(tf.keras.layers.Layer):
    def __init__(self):
        super(Predict, self).__init__()
        self.dense_layer = tf.keras.layers.Dense(512, activation='relu', name="full connection", dtype=tf.float64)
        self.predict = tf.keras.layers.Dense(1, name="prediction", dtype=tf.float64)

    def call(self, inputs, **kwargs):
        print(type(inputs), len(inputs))
        middle = self.dense_layer(inputs[0] - inputs[1])
        prediction = self.predict(middle)
        return tf.abs(prediction)

上面定义的两个层，第一个层是作为 Batch MaskMean 层，Predict层是回归模型的输出层

这里注意：

1
2
mask_mean_layer = MaskMean()
output_layer = Predict()

这里实例化了两个层的对象，但是此时 mask_mean_layer.trainable_variables 和 output_layer.trainable_variables 为空，这是 tf.keras 的一个特点，因为此时不知道 input_shape, 所以此时还没有分配层的变量，只有当输入一次数据后才会产生变量，当输入一次数据后， trainable_variables 参数就已经产生了。

自定义模型

另外的注意点：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
class Regulation(tf.keras.Model):
    def __init__(self):
        super(Regulation, self).__init__()
        self.predict = Predict()
        self.mask_mean_layer = MaskMean()

    def call(self, inputs, sentence_pair=False, **kwargs):
        if sentence_pair:
            prediction = self.sentence_pair(inputs, **kwargs)
        else:
            prediction = self.sentence(inputs, **kwargs)
        return prediction

    def sentence(self, inputs, **kwargs):
        """
        the input of model is one sentence and this function is
        to split the data into input_ids, input_mask, segment_ids
        :param inputs: dataset
        :param kwargs: config of input, {sentence_pair:bool, training:bool}
        :return: input_ids, input_mask, segment_ids
        """
        input_ids = inputs[0]
        if len(inputs) >= 2:
            input_mask = inputs[1]
        if len(inputs) >= 3:
            segment_ids = inputs[2]
            # [batch_size, max_seq_len, embedding_size]
            embeddings = self.embedding((input_ids, input_mask, segment_ids), **kwargs)[0]
            # Dropout keep shape
            dropout = self.dropout(inputs=embeddings)
            # [batch_size, max_seq_len, embedding_size] -> [batch_size, seq_len, embedding_size]
            # seq_a_len may be not equal to seq_b_len
            mask = tf.boolean_mask(dropout, input_mask, axis=1)
            # [batch_size, seq_len, embedding_size] -> [batch_size, embedding_size]
            # let all token embeddings' mean as the sentence embedding
            mean = tf.reduce_mean(mask, 1)
            # [batch_size, embedding_size] -> [batch_size*embedding_size,]
            flatten = tf.reshape(mean, (-1,))
            middle = self.inmediate(flatten)
            prediction = tf.reshape(self.predict(middle), [-1, ])
            return tf.abs(prediction)

    def sentence_pair(self, inputs, **kwargs):
        """
        the input of model is sentence pairs and this function is
        to split the data into input_ids, input_mask, segment_ids
        :param inputs: dataset
        :param kwargs: config of input, {'sentence_pair:bool, training:bool}
        :return: input_ids:(sentence_a, sentence_b), input_mask:(input_mask_a, input_mask_b), segment_ids:(segment_ids_a, segment_ids_b)
        """
        # input_ids = inputs[0]
        embeddings = inputs[0]
        if len(inputs) >= 2:
            input_mask = inputs[1]
        if len(inputs) >= 3:
            segment_ids = inputs[2]
        mean_a, mean_b = self.mask_mean_layer((embeddings[0], input_mask[0]), **kwargs), self.mask_mean_layer(
            (embeddings[1], input_mask[1]), **kwargs)
        prediction = self.predict((mean_a, mean_b), **kwargs)
        return prediction

model = Regulation()

这里定义了一个 tf.keras.Model 的一个子类，包含两个自定义层。这时查看 model.layers 可以发现只有一个层， 而且即使输入一次数据，它的 trainable_variables 仍为空，后来发现 transformers 库里的每个 Model 都只包含一个层，所以经过尝试，重新定义如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
class Regulation(tf.keras.layers.Layer):
    def __init__(self):
        super(Regulation, self).__init__()
        self.predict = Predict()
        self.mask_mean_layer = MaskMean()

    def call(self, inputs, sentence_pair=False, **kwargs):
        if sentence_pair:
            prediction = self.sentence_pair(inputs, **kwargs)
        else:
            prediction = self.sentence(inputs, **kwargs)
        return prediction

    def sentence(self, inputs, **kwargs):
        """
        the input of model is one sentence and this function is
        to split the data into input_ids, input_mask, segment_ids
        :param inputs: dataset
        :param kwargs: config of input, {sentence_pair:bool, training:bool}
        :return: input_ids, input_mask, segment_ids
        """
        input_ids = inputs[0]
        if len(inputs) >= 2:
            input_mask = inputs[1]
        if len(inputs) >= 3:
            segment_ids = inputs[2]
            # [batch_size, max_seq_len, embedding_size]
            embeddings = self.embedding((input_ids, input_mask, segment_ids), **kwargs)[0]
            # Dropout keep shape
            dropout = self.dropout(inputs=embeddings)
            # [batch_size, max_seq_len, embedding_size] -> [batch_size, seq_len, embedding_size]
            # seq_a_len may be not equal to seq_b_len
            mask = tf.boolean_mask(dropout, input_mask, axis=1)
            # [batch_size, seq_len, embedding_size] -> [batch_size, embedding_size]
            # let all token embeddings' mean as the sentence embedding
            mean = tf.reduce_mean(mask, 1)
            # [batch_size, embedding_size] -> [batch_size*embedding_size,]
            flatten = tf.reshape(mean, (-1,))
            middle = self.inmediate(flatten)
            prediction = tf.reshape(self.predict(middle), [-1, ])
            return tf.abs(prediction)

    def sentence_pair(self, inputs, **kwargs):
        """
        the input of model is sentence pairs and this function is
        to split the data into input_ids, input_mask, segment_ids
        :param inputs: dataset
        :param kwargs: config of input, {'sentence_pair:bool, training:bool}
        :return: input_ids:(sentence_a, sentence_b), input_mask:(input_mask_a, input_mask_b), segment_ids:(segment_ids_a, segment_ids_b)
        """
        embeddings = inputs[0]
        if len(inputs) >= 2:
            input_mask = inputs[1]
        if len(inputs) >= 3:
            segment_ids = inputs[2]
        mean_a, mean_b = self.mask_mean_layer((embeddings[0], input_mask[0]), **kwargs), self.mask_mean_layer(
            (embeddings[1], input_mask[1]), **kwargs)
        prediction = self.predict((mean_a, mean_b), **kwargs)
        return prediction

class RegulateModel(tf.keras.Model):
    def __init__(self):
        super(RegulateModel, self).__init__()
        self.regulation_layer = Regulation()

    def call(self, inputs, sentence_pair=True, **kwargs):
        return self.regulation_layer(inputs, sentence_pair=sentence_pair)

model = RegulateModel()

这时可以发现 model.layers 包含一个层， 而且model.layers[0]._layers 包含两个自定义层，此时输入一次数据并输出后，发现 model.trainable_variable 已经分配变量了。

结论： 通过这次试验发现，tf.keras.Model 的子类只能包含一个层，而且只有自定义的层可以包含多个网络层，而且只有在输入一次数据后才会分配变量。

赏

谢谢你请我吃糖果

扫一扫，分享到微信

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia-plus根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

主要涉及技术：
自然语言处理、
Linux

Python

联系QQ:1406332752

很惭愧

只做了一点微小的工作
谢谢大家