LongLong's Blog

k = 0;
#对于每个分类的地址将获取到的问题写入一个文件
for url in urls:
    k += 1;
    fp = open(str(k) + '.txt', 'w');
    #获取该分类下的问题总数
    request = urllib.request.Request(url = url,  headers = headers);
    try:
        res = urllib.request.urlopen(request, timeout=3);
        html = res.read().decode('gbk');
    except Exception as e:
        print(e);
        continue;
    #计算页数
    num = int(jQuery(html).find('span.count-num')[0].text);
    page = num // size;

    #按页进行数据获取
    for i in range(0, page):
        request = urllib.request.Request(url = url + '&rn=' + str(size) + '&pn=' + str(i * size),  headers = headers);
        #如果出现错误就放弃当页的数据，防止整个程序退出
        try:
            res = urllib.request.urlopen(request, timeout=3);
            html = res.read().decode('gbk');
        except Exception as e:
            print(e);
            continue;
        #通过pyquery获取保存问题标题的a标签
        for x in jQuery(html).find('a.title-link'):
            fp.write(x.text.strip() + "\n");
        time.sleep(2);
    fp.close();

import jieba.analyse;
import numpy;
import pickle;
import tensorflow.keras;
import scipy.sparse;
from tensorflow.keras.layers import Dense, Activation;
if __name__ == '__main__':
    category = 20;
    k = 0;
    j = 0;
    #生成词汇表
    words = dict();
    for i in range(0, category):
        #对于每个文件（该分类）中的问题
        fp = open(str(i + 1) + '.txt');
        for x in fp:
            j += 1;
            #进行TF-IDF分词，IDF使用分词引擎中的数据
            for y in jieba.analyse.extract_tags(x, withWeight=True):
                #如果是新词，则更新词汇表
                if y[0] not in words:
                    words[y[0]] = k;
                    k += 1;
        fp.close();
    #将词汇表保存，以便于预测时使用
    pickle.dump(words, open('words.bin', 'wb'));
    #确定了两个数组的维度，这里使用稀疏矩阵可以节约一定的内存空间
    train_x = scipy.sparse.lil_matrix((j, len(words)));
    train_y = numpy.zeros((j, 1));
    j = 0;
    #将分类索引值和TF-IDF值填入对应的数组中
    for i in range(0, category):
        fp = open(str(i + 1) + '.txt');
        for x in fp:
            #填入分类索引值
            train_y[j] = i;
            for y in jieba.analyse.extract_tags(x, withWeight=True):
                #填入TF-IDF
                train_x[j, words[y[0]]] = y[1];
            j = j + 1;
    #初始化模型
    model = tensorflow.keras.Sequential();
    #设置网络
    model.add(Dense(32, activation='relu', input_dim=len(words)));
    model.add(Dense(category, activation='sigmoid'));
    #设置分类的优化器
    model.compile(optimizer='rmsprop',
        loss='categorical_crossentropy',
        metrics=['accuracy']);
    #处理分类索引值数组
    train_y = tensorflow.keras.utils.to_categorical(train_y, num_classes=category);
    #开始训练
    model.fit(train_x, train_y, epochs=10, batch_size=32);
    #保存训练得到的模型
    model.save('1.h5');

#读取词库
words = pickle.load(open('words.bin', 'rb'));
#加载模型
model = load_model('1.h5');
#需要预测的文本信息，由命令行参数给出
text = sys.argv[1];
#输入向量的维度为1x词汇表长度
data = numpy.zeros((1, len(words)));
#分词并计算TF-IDF
for x in jieba.analyse.extract_tags(text, withWeight=True):
    #如果词汇表中有此词汇则将TF-IDF填入输入向量中
    if x[0] in words:
        data[0, words[x[0]]] = x[1];
#预测，并取出可能性最高的一个分类的索引值
i = numpy.argmax(model.predict(data));
print(labels[i]);