import jieba
import torch
from jieba import lcut
from openpyxl.styles.builtins import output
from tensorflow.keras.preprocessing.text import Tokenizer
from torch.utils.tensorboard import SummaryWriter
import torch.nn as nndef dm_embedding_show():# 1.对句子分词 word.listsentence1 = '传智教育是一家上市公司，旗下有黑马程序员品牌。我是在黑马这里学习人工智能'sentence2 = '我爱自然语言处理'sentences = [sentence1, sentence2]# 2.对所有句子进行分词word_list = list()for s in sentences:word_list.append(lcut(s))print(f'word_list---》{word_list}')# 3.获取word_index,index_wordtokenize = Tokenizer()tokenize.fit_on_texts(word_list)print(f'tokenize.index_word---》{tokenize.index_word}')print(f'tokenize.word_index---》{tokenize.word_index}')# 4.将文本序列转换成数字序列seq_ids = tokenize.texts_to_sequences(word_list)print(f'seq_ids---》{seq_ids}')# 5.获取样本中的所有单词words = tokenize.word_index.keys()print(f'words---》{words}')# 6.实例化Embedding对象# 6.1 num_embeddings:代表需要进行词向量表示的单词总个数（一定是去重）# 6.2 embedding_dim:代表每个单词的词向量维度embed = nn.Embedding(len(words), 8)print(f'embed.weight.shape---》{embed.weight.shape}')print(f'embed.weight.data--》{embed.weight.data}')# 7.可视化embeddingsummary = SummaryWriter()summary.add_embedding(embed.weight.data,metadata=list(words))summary.close()# 8.获取每个单词对应的词向量for idx in range(len(tokenize.index_word)):output = embed(torch.tensor([idx]))print(f'{tokenize.index_word[idx+1]}的词向量是{output}')if __name__ == '__main__':dm_embedding_show()
