1 # coding:utf-8
 2 import re
 3 import json
 4 import jieba.posseg as pseg
 5 import string
 6 import datetime
 7 import zhon.hanzi
 8 import get_comment.SQL
 9 
10 #   要清洗掉的中文標點
11 ignoring_words = list(zhon.hanzi.punctuation)+list(string.punctuation)
12 #   數據庫表中沒有存儲評論的表名稱
13 ignoring_table = ['phone_info', 'phone_url', 'error_table']
14 
15 
16 #   將分詞后的數據儲存到本地
17 def text_save(content, filename, mode='a'):
18     file = open(filename, mode)
19     file.write(str(content))    # 存儲為str格式
20     # 此處為存儲為json格式
21     # js = json.dumps(content)
22     # file.write(js)
23     file.close()
24 
25 
26 #   獲取數據庫中的評論
27 def get_comments(phones_id, i):
28     mysql = get_comment.SQL.save_mysql()    # 調用數據庫操作
29     comment = mysql.read_comment_phone(phones_id, i)
30     for ob in comment:
31         return str((ob[0]))
32 
33 
34 #   調用jieba分詞包進行分詞
35 def jieba_cut(phones_id, i):
36     comment = get_comments(phones_id, i)
37     comments_dict = dict(pseg.cut(comment))
38     return comments_dict
39 
40 
41 #   獲取數據庫中的所有儲存評論信息的表名
42 def get_phone_table():
43     mysql = get_comment.SQL.save_mysql()
44     phone_id_list = list(mysql.select_all_table('jd'))
45     for table in phone_id_list:
46         if table[0] in ignoring_table:
47             phone_id_list.remove(table)
48     phone_id_list.pop()
49     return phone_id_list
50 
51 
52 #   調用其他函數完成數據的存儲，格式為txt
53 def save_file(phone_id1):
54     with open('E:\\PythonFile\\tingyongci.txt') as ti:
55         ti_list = list(ti.read())  # 獲取停用詞表（綜合哈工大停用詞詞表）
56     for i in range(1, 1000):
57         with open('E:\\PythonFile\\tingyongci.txt') as ti:
58             ti_list = list(ti.read())  # 獲取停用詞表（綜合哈工大停用詞詞表）
59         for i in range(1, 1000):
60             try:
61                 new_dic = {}
62                 new_dic1 = jieba_cut(phone_id1, i)
63                 for key, value in new_dic1.items():
64                     if key not in ti_list:
65                         new_dic[key] = value
66                     else:
67                         pass
68             except:
69                 continue
70             print(new_dic)
71             text_save(new_dic, 'E:\\PythonFile\\jd\\phone\\%s.txt' % phone_id1)
72     print('----------------------id=%s---------------------------' % phone_id)
73 
74 
75 if __name__ == "__main__":
76         start_time = datetime.datetime.now()
77         phone_table_lists = get_phone_table()
78         for phone_table_name in phone_table_lists:
79             phone_id = re.sub("\D", "", phone_table_name[0])    # 獲取表名中的phone_id
80             save_file(int(phone_id))
81             print("--------------id=%s--------------" % phone_id)
82         end_time = datetime.datetime.now()
83         print('運行時間：')
84         print(end_time - start_time)