diff --git "a/2\345\260\217\346\227\266\345\205\245\351\227\250TensorFlow\347\274\226\347\250\213\345\205\254\345\274\200\350\257\276/\350\257\246\347\273\206\346\263\250\350\247\243\346\272\220\347\240\201/data/dgk_shooter_z.conv.zip" "b/2\345\260\217\346\227\266\345\205\245\351\227\250TensorFlow\347\274\226\347\250\213\345\205\254\345\274\200\350\257\276/\350\257\246\347\273\206\346\263\250\350\247\243\346\272\220\347\240\201/data/dgk_shooter_z.conv.zip" deleted file mode 100644 index 7301fd7..0000000 Binary files "a/2\345\260\217\346\227\266\345\205\245\351\227\250TensorFlow\347\274\226\347\250\213\345\205\254\345\274\200\350\257\276/\350\257\246\347\273\206\346\263\250\350\247\243\346\272\220\347\240\201/data/dgk_shooter_z.conv.zip" and /dev/null differ diff --git a/lessonOne/imgClassifierWeb/__pycache__/cnnModel.cpython-36.pyc b/lessonOne/imgClassifierWeb/__pycache__/cnnModel.cpython-36.pyc index c20a01d..00d570a 100644 Binary files a/lessonOne/imgClassifierWeb/__pycache__/cnnModel.cpython-36.pyc and b/lessonOne/imgClassifierWeb/__pycache__/cnnModel.cpython-36.pyc differ diff --git a/lessonOne/imgClassifierWeb/__pycache__/execute.cpython-36.pyc b/lessonOne/imgClassifierWeb/__pycache__/execute.cpython-36.pyc index e69d6b3..0034063 100644 Binary files a/lessonOne/imgClassifierWeb/__pycache__/execute.cpython-36.pyc and b/lessonOne/imgClassifierWeb/__pycache__/execute.cpython-36.pyc differ diff --git a/lessonOne/imgClassifierWeb/__pycache__/getConfig.cpython-36.pyc b/lessonOne/imgClassifierWeb/__pycache__/getConfig.cpython-36.pyc index c7008e4..a292b39 100644 Binary files a/lessonOne/imgClassifierWeb/__pycache__/getConfig.cpython-36.pyc and b/lessonOne/imgClassifierWeb/__pycache__/getConfig.cpython-36.pyc differ diff --git a/lessonOne/imgClassifierWeb/airplane.png b/lessonOne/imgClassifierWeb/airplane.png new file mode 100644 index 0000000..8916132 Binary files /dev/null and b/lessonOne/imgClassifierWeb/airplane.png differ diff --git a/lessonOne/imgClassifierWeb/app.py b/lessonOne/imgClassifierWeb/app.py old mode 100755 new mode 100644 index b6a09d3..7d8a266 --- a/lessonOne/imgClassifierWeb/app.py +++ b/lessonOne/imgClassifierWeb/app.py @@ -1,10 +1,3 @@ -""" -Good and quick. -Deeper in ML and DL. -Overfitting. -Github readme instructions should tell only how to run the code. -""" - import flask import werkzeug import os @@ -12,14 +5,15 @@ import tensorflow as tf import getConfig import execute -gConfig={} -gConfig=getConfig.get_config(config_file='config.ini') -#Creating a new Flask Web application. It accepts the package name. +gConfig = {} +gConfig = getConfig.get_config(config_file='config.ini') + +# Creating a new Flask Web application. It accepts the package name. app = flask.Flask("imgClassifierWeb") -def CNN_predict(): +def CNN_predict(): global sess global model global graph @@ -28,13 +22,13 @@ def CNN_predict(): global: """ global secure_filename - #从本地目录读取需要分类的图片 + # 从本地目录读取需要分类的图片 img = scipy.misc.imread(os.path.join(app.root_path, secure_filename)) """ 校验图片格式 """ - if(img.ndim) == 3: + if (img.ndim) == 3: """ 是否为32*32 """ @@ -44,13 +38,12 @@ def CNN_predict(): """ if img.shape[-1] == 3: - - - predicted_class = execute.predict_line(sess,model,img,graph) + predicted_class = execute.predict_line(sess, model, img, graph) """ 将返回的结果用页面模板给渲染出来 """ - return flask.render_template(template_name_or_list="prediction_result.html", predicted_class=predicted_class) + return flask.render_template(template_name_or_list="prediction_result.html", + predicted_class=predicted_class) else: """ 如果检测出图片格式不符合要求，则返回错误并返回上传图片的格式""" return flask.render_template(template_name_or_list="error.html", img_shape=img.shape) @@ -58,98 +51,80 @@ def CNN_predict(): """ 如果检测出图片格式不符合要求，则返回错误并返回上传图片的格式""" return flask.render_template(template_name_or_list="error.html", img_shape=img.shape) return "遇到非图片格式的未知错误，请联系技术人员解决" + + """ flask路由系统： - 1、使用flask.Flask.route() 修饰器。 2、使用flask.Flask.add_url_rule()函数。 - 3、直接访问基于werkzeug路由系统的flask.Flask.url_map. - 参考知识链接：https://www.jianshu.com/p/e69016bd8f08 - 1、@app.route('/index.html') def index(): return "Hello World!" - 2、def index(): return "Hello World!" index = app.route('/index.html')(index) - - - app.add_url_rule:app.add_url_rule(rule,endpoint,view_func) - 关于rule、ednpoint、view_func以及函数注册路由的原理可以参考：https://www.cnblogs.com/eric-nirnava/p/endpoint.html - """ app.add_url_rule(rule="/predict/", endpoint="predict", view_func=CNN_predict) """ 知识点： flask.request属性 - form: 一个从POST和PUT请求解析的 MultiDict（一键多值字典）。 - args: MultiDict，要操作 URL （如 ?key=value ）中提交的参数可以使用 args 属性: - searchword = request.args.get('key', '') - values: CombinedMultiDict，内容是form和args。 可以使用values替代form和args。 - cookies: 顾名思义，请求的cookies，类型是dict。 - stream: 在可知的mimetype下，如果进来的表单数据无法解码，会没有任何改动的保存到这个·stream·以供使用。很多时候，当请求的数据转换为string时，使用data是最好的方式。这个stream只返回数据一次。 - headers: 请求头，字典类型。 - data: 包含了请求的数据，并转换为字符串，除非是一个Flask无法处理的mimetype。 - files: MultiDict，带有通过POST或PUT请求上传的文件。 - method: 请求方法，比如POST、GET - 知识点参考链接：https://blog.csdn.net/yannanxiu/article/details/53116652 - - - werkzeug - """ + + def upload_image(): global secure_filename - if flask.request.method == "POST":#设置request的模式为POST - img_file = flask.request.files["image_file"]#获取需要分类的图片 - secure_filename = werkzeug.secure_filename(img_file.filename)#生成一个没有乱码的文件名 - img_path = os.path.join(app.root_path, secure_filename)#获取图片的保存路径 - img_file.save(img_path)#将图片保存在应用的根目录下 + if flask.request.method == "POST": # 设置request的模式为POST + img_file = flask.request.files["image_file"] # 获取需要分类的图片 + secure_filename = werkzeug.secure_filename(img_file.filename) # 生成一个没有乱码的文件名 + img_path = os.path.join(app.root_path, secure_filename) # 获取图片的保存路径 + img_file.save(img_path) # 将图片保存在应用的根目录下 print("图片上传成功.") """ - + """ return flask.redirect(flask.url_for(endpoint="predict")) return "图片上传失败" + """ """ app.add_url_rule(rule="/upload/", endpoint="upload", view_func=upload_image, methods=["POST"]) -def redirect_upload(): +def redirect_upload(): return flask.render_template(template_name_or_list="upload_image.html") + + """ """ app.add_url_rule(rule="/", endpoint="homepage", view_func=redirect_upload) sess = tf.Session() -sess, model,graph = execute.init_session(sess, conf='config.ini') +sess, model, graph = execute.init_session(sess, conf='config.ini') if __name__ == "__main__": app.run(host="localhost", port=7777, debug=False) diff --git a/lessonOne/imgClassifierWeb/cnnModel.py b/lessonOne/imgClassifierWeb/cnnModel.py index 108d5c6..1196aaf 100644 --- a/lessonOne/imgClassifierWeb/cnnModel.py +++ b/lessonOne/imgClassifierWeb/cnnModel.py @@ -5,6 +5,7 @@ import numpy as np import pickle import getConfig +from collections import Counter gConfig={} @@ -33,6 +34,7 @@ def __init__(self,percent,learning_rate,learning_rate_decay_factor): self.percent=percent self.learning_rate=tf.Variable(float(learning_rate), trainable=False) self.learning_rate_decay_op = self.learning_rate.assign(self.learning_rate * learning_rate_decay_factor) + self.global_step = tf.Variable(0, trainable=False) def create_conv_layer(input_data, filter_size, num_filters): filters = tf.Variable(tf.truncated_normal(shape=( @@ -119,8 +121,10 @@ def fc_layer(flattened_layer, num_inputs, num_outputs): labels=self.label_tensor) cost=tf.reduce_mean(cross_entropy) - self.ops=tf.train.GradientDescentOptimizer(self.learning_rate).minimize(cost) - #保存所有变量的值 + self.ops=tf.train.GradientDescentOptimizer(self.learning_rate).minimize(cost,global_step=self.global_step) + #保存所有变量的 + sess=tf.Session() + sess.run(tf.global_variables_initializer()) self.saver = tf.train.Saver(tf.all_variables()) def step(self,sess,shuffled_data,shuffled_labels,graph,forward_only=None): @@ -134,6 +138,9 @@ def step(self,sess,shuffled_data,shuffled_labels,graph,forward_only=None): k_size=gConfig['percent']*gConfig['dataset_size']/100 dataset_array = np.random.rand(int(k_size), 32, 32, 3) dataset_array[0,:,:,:] = shuffled_data + print(shuffled_data) + print(dataset_array[0]) + print(dataset_array) feed_dict_test={data_tensor:dataset_array,keep_prop:1.0 } softmax_propabilities_, softmax_predictions_ = sess.run([self.softmax_propabilities, self.softmax_predictions], @@ -143,10 +150,16 @@ def step(self,sess,shuffled_data,shuffled_labels,graph,forward_only=None): patch_bin_file = open(file, 'rb') label_names_dict = pickle.load(patch_bin_file) print(label_names_dict) + print(softmax_predictions_[0]) + print(softmax_predictions_) + print(Counter(softmax_predictions_).most_common(1)) + + #k=Counter(softmax_predictions_).most_common(1) + #print(k) dataset_label_names = label_names_dict["label_names"] return dataset_label_names[softmax_predictions_[0]] else: - + cnn_feed_dict = {self.data_tensor: shuffled_data, self.label_tensor: shuffled_labels, keep_prop: gConfig['keeps']} softmax_predictions_, _ = sess.run([self.softmax_predictions, self.ops],feed_dict=cnn_feed_dict) # 统计预测争取的数量 diff --git a/lessonOne/imgClassifierWeb/config.ini b/lessonOne/imgClassifierWeb/config.ini index 05de072..a0c48d2 100644 --- a/lessonOne/imgClassifierWeb/config.ini +++ b/lessonOne/imgClassifierWeb/config.ini @@ -2,7 +2,8 @@ # Mode : train, test, serve mode = train working_directory = model/ -dataset_path=/Users/zhaoyingjun/Learning/TensorFlow-Coding/lessionOne/imgClassifierWeb/train_data/ +dataset_path=/Users/zhaoyingjun/Learning/TensorFlow_code/lessonOne/imgClassifierWeb/train_data/ +dataset_test=/Users/zhaoyingjun/Learning/TensorFlow_code/lessonOne/imgClassifierWeb/test_data/ [ints] steps_per_checkpoint = 10 diff --git a/lessonOne/imgClassifierWeb/deer.png b/lessonOne/imgClassifierWeb/deer.png new file mode 100644 index 0000000..1e3af0f Binary files /dev/null and b/lessonOne/imgClassifierWeb/deer.png differ diff --git a/lessonOne/imgClassifierWeb/dog.png b/lessonOne/imgClassifierWeb/dog.png new file mode 100644 index 0000000..b234d55 Binary files /dev/null and b/lessonOne/imgClassifierWeb/dog.png differ diff --git a/lessonOne/imgClassifierWeb/execute.py b/lessonOne/imgClassifierWeb/execute.py index 303deb2..8451944 100644 --- a/lessonOne/imgClassifierWeb/execute.py +++ b/lessonOne/imgClassifierWeb/execute.py @@ -9,7 +9,9 @@ gConfig = {} def read_data(dataset_path, im_dim, num_channels,num_files,images_per_file): - files_names = os.listdir(dataset_path) # 获取训练集中训练文件的名称 + files_names = os.listdir(dataset_path) + print(files_names) + # 获取训练集中训练文件的名称 """ 在CIFAR10中已经为我们标注和准备好了数据，一时找不到合适的高质量的标注训练集，我们就是使用CIFAR10的来作为我们的训练集。 在训练集中一共有50000个训练样本，放到5个二进制文件中心，每个样本有3072个像素点，是32*3维度的 @@ -22,7 +24,7 @@ def read_data(dataset_path, im_dim, num_channels,num_files,images_per_file): #从训练集中读取二进制数据并将其维度转换成32*32*3 for file_name in files_names: - if file_name[0:len(file_name) - 1] == "data_batch_": + if file_name[0:len(file_name)-1] == "data_batch_": print("正在处理数据 : ", file_name) data_dict = unpickle_patch(dataset_path + file_name) images_data = data_dict[b"data"] @@ -71,6 +73,7 @@ def get_batch(data,labels,percent): np.random.shuffle(shuffled_labels) return data[shuffled_labels[:num_elements], :, :, :], shuffled_labels[:num_elements] + #定义训练函数 def train(): """使用BFC内存管理管理算法，tf.ConfigProto()用于GPU的管理，可以控制GPU的使用率 @@ -92,7 +95,10 @@ def train(): dataset_array, dataset_labels = read_data(dataset_path=gConfig['dataset_path'], im_dim=gConfig['im_dim'], - num_channels=gConfig['num_channels'],num_files=gConfig['num_files'],images_per_file=gConfig['images_per_file']) + num_channels=gConfig['num_channels'],num_files=gConfig['num_files'],images_per_file=gConfig['images_per_file']) + + + dataset_array_test, dataset_labels_test = read_data(dataset_path=gConfig['dataset_test'], im_dim=gConfig['im_dim'], num_channels=gConfig['num_channels'],num_files=1,images_per_file=gConfig['images_per_file']) print("Size of data : ", dataset_array.shape) with tf.Session(config=config) as sess: model,_=create_model(sess,False) @@ -100,9 +106,17 @@ def train(): step_time, accuracy = 0.0, 0.0 current_step = 0 previous_correct = [] - shuffled_data, shuffled_labels = get_batch(data=dataset_array, labels=dataset_labels, - percent=gConfig['percent']) + while model.learning_rate.eval()>gConfig['end_learning_rate']: + + shuffled_data, shuffled_labels = get_batch(data=dataset_array, labels=dataset_labels, + percent=gConfig['percent']) + #print(shuffled_data) + + shuffled_data_test, shuffled_labels_test = get_batch(data=dataset_array_test, labels=dataset_labels_test, + percent=5*gConfig['percent']) + + start_time = time.time() step_correct=model.step(sess,shuffled_data,shuffled_labels,False) step_time += (time.time() - start_time) / gConfig['steps_per_checkpoint'] @@ -116,13 +130,36 @@ def train(): sess.run(model.learning_rate_decay_op) previous_correct.append(accuracy) checkpoint_path = os.path.join(gConfig['working_directory'], "cnn.ckpt") - model.saver.save(sess, checkpoint_path) - print("在", str(gConfig['percent'] *gConfig['dataset_size']/100),"个样本集上训练的准确率", ' : ', accuracy) + #saver=tf.train.Saver() + model.saver.save(sess, checkpoint_path,global_step=model.global_step) + + #sess.run(tf.global_variables_initializer()) + #以下为增加模型在测试集上的准确率测试 + graph = tf.get_default_graph() + + softmax_propabilities = graph.get_tensor_by_name(name="softmax_probs:0") + softmax_predictions = tf.argmax(softmax_propabilities, axis=1) + data_tensor = graph.get_tensor_by_name(name="data_tensor:0") + label_tensor = graph.get_tensor_by_name(name="label_tensor:0") + keep_prop = graph.get_tensor_by_name(name="keep_prop:0") + + feed_dict_testing = {data_tensor: shuffled_data_test, + label_tensor: shuffled_labels_test, + keep_prop: 1.0} + + softmax_propabilities_, softmax_predictions_ = sess.run([softmax_propabilities, softmax_predictions], + feed_dict=feed_dict_testing) + + correct = np.array(np.where(softmax_predictions_ == shuffled_labels_test)) + correct = correct.size + print("模型在测试集上的准确率为 : ", correct/(gConfig['percent']*gConfig['dataset_size']/100)) + + + print("在", str(gConfig['percent'] *gConfig['dataset_size']/100),"个训练集上训练的准确率", ' : ', accuracy) print("学习率 %.4f 每步耗时 %.2f " % ( model.learning_rate.eval(),step_time)) step_time, accuracy = 0.0,0.0 sys.stdout.flush() - def init_session(sess,conf='config.ini'): global gConfig gConfig=getConfig.get_config(conf) diff --git a/lessonOne/imgClassifierWeb/im.png b/lessonOne/imgClassifierWeb/im.png new file mode 100644 index 0000000..0df1a2e Binary files /dev/null and b/lessonOne/imgClassifierWeb/im.png differ diff --git a/lessonOne/imgClassifierWeb/model/checkpoint b/lessonOne/imgClassifierWeb/model/checkpoint deleted file mode 100644 index 157fd12..0000000 --- a/lessonOne/imgClassifierWeb/model/checkpoint +++ /dev/null @@ -1,2 +0,0 @@ -model_checkpoint_path: "cnn.ckpt" -all_model_checkpoint_paths: "cnn.ckpt" diff --git a/lessonOne/imgClassifierWeb/model/cnn.ckpt.data-00000-of-00001 b/lessonOne/imgClassifierWeb/model/cnn.ckpt.data-00000-of-00001 deleted file mode 100644 index bc32101..0000000 Binary files a/lessonOne/imgClassifierWeb/model/cnn.ckpt.data-00000-of-00001 and /dev/null differ diff --git a/lessonOne/imgClassifierWeb/model/cnn.ckpt.index b/lessonOne/imgClassifierWeb/model/cnn.ckpt.index deleted file mode 100644 index e76dfab..0000000 Binary files a/lessonOne/imgClassifierWeb/model/cnn.ckpt.index and /dev/null differ diff --git a/lessonOne/imgClassifierWeb/model/cnn.ckpt.meta b/lessonOne/imgClassifierWeb/model/cnn.ckpt.meta deleted file mode 100644 index 451e849..0000000 Binary files a/lessonOne/imgClassifierWeb/model/cnn.ckpt.meta and /dev/null differ diff --git a/lessonOne/imgClassifierWeb/train_data/test_batch b/lessonOne/imgClassifierWeb/test_data/data_batch_6 similarity index 100% rename from lessonOne/imgClassifierWeb/train_data/test_batch rename to lessonOne/imgClassifierWeb/test_data/data_batch_6 diff --git a/lessonThree/Anti-Fraud-App/VAE.py b/lessonThree/Anti-Fraud-App/VAE.py new file mode 100644 index 0000000..6eb0d97 --- /dev/null +++ b/lessonThree/Anti-Fraud-App/VAE.py @@ -0,0 +1,96 @@ +import tensorflow as tf +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from tensorflow.examples.tutorials.mnist import input_data +import getConfig +gConfig={} +gConfig=getConfig.get_config(config_file='config.ini') +#定义训练数据的维度 +seq_len=gConfig['seqlen'] + +#读取数据 +def read_data(source_file): + data=pd.read_csv(source_file,encoding='utf-8') + dataset=data.values + return dataset +dataset = read_data(gConfig['input_file']) + +tf.reset_default_graph() + +batch_size = 64 +X_in = tf.placeholder(dtype=tf.float32, shape=[None, seq_len], name='X') +Y = tf.placeholder(dtype=tf.float32, shape=[None, seq_len], name='Y') +Y_flat = tf.reshape(Y, shape=[-1, seq_len * 1]) +keep_prob = tf.placeholder(dtype=tf.float32, shape=(), name='keep_prob') + +dec_in_channels = 1 +n_latent = 8 +reshaped_dim = [-1, 7, 7, dec_in_channels] +inputs_decoder = 49 * dec_in_channels // 2 + +#定义激活函数 + +def lrelu(x, alpha=0.3): + return tf.maximum(x, tf.multiply(x, alpha)) +#定义编码机 +def encoder(X_in, keep_prob): + activation = lrelu + with tf.variable_scope("encoder", reuse=None): + X = tf.reshape(X_in, shape=[-1, seq_len, 1, 1]) + x = tf.layers.conv2d(X, filters=64, kernel_size=4, strides=2, padding='same', activation=activation) + x = tf.nn.dropout(x, keep_prob) + x = tf.layers.conv2d(x, filters=64, kernel_size=4, strides=2, padding='same', activation=activation) + x = tf.nn.dropout(x, keep_prob) + x = tf.layers.conv2d(x, filters=64, kernel_size=4, strides=1, padding='same', activation=activation) + x = tf.nn.dropout(x, keep_prob) + x = tf.contrib.layers.flatten(x) + mn = tf.layers.dense(x, units=n_latent) + sd = 0.5 * tf.layers.dense(x, units=n_latent) + epsilon = tf.random_normal(tf.stack([tf.shape(x)[0], n_latent])) + z = mn + tf.multiply(epsilon, tf.exp(sd)) + return z, mn, sd + +#定义解码机 +def decoder(sampled_z, keep_prob): + with tf.variable_scope("decoder", reuse=None): + x = tf.layers.dense(sampled_z, units=inputs_decoder, activation=lrelu) + x = tf.layers.dense(x, units=inputs_decoder * 2 + 1, activation=lrelu) + x = tf.reshape(x, reshaped_dim) + x = tf.layers.conv2d_transpose(x, filters=64, kernel_size=4, strides=2, padding='same', activation=tf.nn.relu) + x = tf.nn.dropout(x, keep_prob) + x = tf.layers.conv2d_transpose(x, filters=64, kernel_size=4, strides=1, padding='same', activation=tf.nn.relu) + x = tf.nn.dropout(x, keep_prob) + x = tf.layers.conv2d_transpose(x, filters=64, kernel_size=4, strides=1, padding='same', activation=tf.nn.relu) + + x = tf.contrib.layers.flatten(x) + x = tf.layers.dense(x, units=seq_len * 1, activation=tf.nn.sigmoid) + decoder_set = tf.reshape(x, shape=[-1, seq_len, 1]) + return decoder_set +#定义计算loss以及进行优化器优化的一系列tensor +sampled, mn, sd = encoder(X_in, keep_prob) +dec = decoder(sampled, keep_prob) + +unreshaped = tf.reshape(dec, [-1, seq_len*1]) +img_loss = tf.reduce_sum(tf.squared_difference(unreshaped, Y_flat), 1) +latent_loss = -0.5 * tf.reduce_sum(1.0 + 2.0 * sd - tf.square(mn) - tf.exp(2.0 * sd), 1) +dst_loss=img_loss+latent_loss +loss = tf.reduce_mean(img_loss + latent_loss) +optimizer = tf.train.AdamOptimizer(gConfig['learning_rate']).minimize(loss) +sess = tf.Session() +sess.run(tf.global_variables_initializer()) + +#开始vae的训练 +for i in range(gConfig['vae_steps']): + batch=dataset + sess.run(optimizer, feed_dict = {X_in: batch, Y: batch, keep_prob: 0.8}) + if not i % 200: + ls, d, i_ls, d_ls, mu, sampled_data = sess.run([loss, dec, img_loss, dst_loss, mn, sampled], feed_dict = {X_in: batch, Y: batch, keep_prob: 1.0}) + + print(i, ls, np.mean(i_ls), np.mean(d_ls)) + +#保存训练的encode的结果，就是要进行特征压缩后的特征 +sampled_data=pd.DataFrame(sampled_data) +sampled_data.to_csv('sampled_data.csv') + + diff --git a/lessonThree/Anti-Fraud-App/__pycache__/data_util.cpython-36.pyc b/lessonThree/Anti-Fraud-App/__pycache__/data_util.cpython-36.pyc new file mode 100644 index 0000000..407d81d Binary files /dev/null and b/lessonThree/Anti-Fraud-App/__pycache__/data_util.cpython-36.pyc differ diff --git a/lessonThree/Anti-Fraud-App/__pycache__/execute.cpython-36.pyc b/lessonThree/Anti-Fraud-App/__pycache__/execute.cpython-36.pyc new file mode 100644 index 0000000..54c1cb0 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/__pycache__/execute.cpython-36.pyc differ diff --git a/lessonThree/Anti-Fraud-App/__pycache__/getConfig.cpython-36.pyc b/lessonThree/Anti-Fraud-App/__pycache__/getConfig.cpython-36.pyc new file mode 100644 index 0000000..ad8281b Binary files /dev/null and b/lessonThree/Anti-Fraud-App/__pycache__/getConfig.cpython-36.pyc differ diff --git a/lessonThree/Anti-Fraud-App/__pycache__/kmeans.cpython-36.pyc b/lessonThree/Anti-Fraud-App/__pycache__/kmeans.cpython-36.pyc new file mode 100644 index 0000000..6c131a0 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/__pycache__/kmeans.cpython-36.pyc differ diff --git a/lessonThree/Anti-Fraud-App/__pycache__/kmeansModel.cpython-36.pyc b/lessonThree/Anti-Fraud-App/__pycache__/kmeansModel.cpython-36.pyc new file mode 100644 index 0000000..f92f847 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/__pycache__/kmeansModel.cpython-36.pyc differ diff --git a/lessonThree/Anti-Fraud-App/app.py b/lessonThree/Anti-Fraud-App/app.py index fd48f2b..8fe391e 100644 --- a/lessonThree/Anti-Fraud-App/app.py +++ b/lessonThree/Anti-Fraud-App/app.py @@ -1,2 +1,30 @@ -import flask +import pandas as pd +import numpy as np +import tensorflow as tf +import os +from flask import Flask, render_template, request, make_response +from flask import jsonify +from flask import Flask +from flask import jsonify +from flask import Response +import sys +import time +import hashlib +import threading +import execute +# +app = Flask(__name__) +#路由注解，我们这里使用的是path的形式进行传参 +@app.route('/predict////////////', methods=['GET']) +def predict(a,b,c,d,e,f,g,h,i,j,k,l): + line=[a,b,c,d,e,f,g,h,i,j,k,l] + lines=range(12) + lines=[int(i) for i in line] + lines=[lines] + predict_result=execute.predicts(lines) + + return jsonify( { 'result of cluster': str(predict_result) } ) + +if (__name__ == "__main__"): + app.run(host = '0.0.0.0', port = 8088) \ No newline at end of file diff --git a/lessonThree/Anti-Fraud-App/autoencoder.py b/lessonThree/Anti-Fraud-App/autoencoder.py index e63b064..9454148 100644 --- a/lessonThree/Anti-Fraud-App/autoencoder.py +++ b/lessonThree/Anti-Fraud-App/autoencoder.py @@ -1,68 +1,88 @@ import tensorflow as tf -import tensorflow as tf import numpy as np -import matplotlib.pyplot as plt - -# 导入MNIST数据 -from tensorflow.examples.tutorials.mnist import input_data -mnist = input_data.read_data_sets("MNIST_data/", one_hot=False) - -learning_rate = 0.01 -training_epochs = 10 -batch_size = 256 -display_step = 1 -examples_to_show = 10 -n_input = 784 - -# tf Graph input (only pictures) -X = tf.placeholder("float", [None, n_input]) - -# 用字典的方式存储各隐藏层的参数 -n_hidden_1 = 256 # 第一编码层神经元个数 -n_hidden_2 = 128 # 第二编码层神经元个数 -# 权重和偏置的变化在编码层和解码层顺序是相逆的 -# 权重参数矩阵维度是每层的 输入*输出，偏置参数维度取决于输出层的单元数 -weights = { - 'encoder_h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])), - 'encoder_h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])), - 'decoder_h1': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_1])), - 'decoder_h2': tf.Variable(tf.random_normal([n_hidden_1, n_input])), -} -biases = { - 'encoder_b1': tf.Variable(tf.random_normal([n_hidden_1])), - 'encoder_b2': tf.Variable(tf.random_normal([n_hidden_2])), - 'decoder_b1': tf.Variable(tf.random_normal([n_hidden_1])), - 'decoder_b2': tf.Variable(tf.random_normal([n_input])), -} - -# 每一层结构都是 xW + b -# 构建编码器 -def encoder(x): - layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['encoder_h1']), - biases['encoder_b1'])) - layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['encoder_h2']), - biases['encoder_b2'])) - return layer_2 - - -# 构建解码器 -def decoder(x): - layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['decoder_h1']), - biases['decoder_b1'])) - layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['decoder_h2']), - biases['decoder_b2'])) - return layer_2 - -# 构建模型 -encoder_op = encoder(X) -decoder_op = decoder(encoder_op) - -# 预测 -y_pred = decoder_op -y_true = X - -# 定义代价函数和优化器 -cost = tf.reduce_mean(tf.pow(y_true - y_pred, 2)) #最小二乘法 -optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost) +import getConfig +import data_util +from functools import partial + +import pandas as pd + +gConfig={} + +gConfig=getConfig.get_config(config_file='config.ini') + +n_inputs = 1 +n_hidden1 = 500 +n_hidden2 = 500 +n_hidden3 = 20 # codings +n_hidden4 = n_hidden2 +n_hidden5 = n_hidden1 +n_outputs = n_inputs +learning_rate = 0.001 + +initializer = tf.contrib.layers.variance_scaling_initializer() +my_dense_layer = partial( + tf.layers.dense, + activation=tf.nn.elu, + kernel_initializer=initializer) + +X = tf.placeholder(tf.float32, [429398, n_inputs]) +hidden1 = my_dense_layer(X, n_hidden1) +hidden2 = my_dense_layer(hidden1, n_hidden2) +hidden3_mean = my_dense_layer(hidden2, n_hidden3, activation=None) +hidden3_gamma = my_dense_layer(hidden2, n_hidden3, activation=None) +noise = tf.random_normal(tf.shape(hidden3_gamma), dtype=tf.float32) +hidden3 = hidden3_mean + tf.exp(0.5 * hidden3_gamma) * noise +hidden4 = my_dense_layer(hidden3, n_hidden4) +hidden5 = my_dense_layer(hidden4, n_hidden5) +logits = my_dense_layer(hidden5, n_outputs, activation=None) +outputs = tf.sigmoid(logits) + +xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits) +reconstruction_loss = tf.reduce_sum(xentropy) +latent_loss = 0.5 * tf.reduce_sum( + tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma) +loss = reconstruction_loss + latent_loss + +optimizer = tf.train.AdamOptimizer(learning_rate=gConfig['learning_rate']) +training_op = optimizer.minimize(reconstruction_loss) + +init = tf.global_variables_initializer() +saver = tf.train.Saver() + + + +n_epochs = 50 +batch_size = 150 +def read_data(source_file): + data=pd.read_csv(source_file,encoding='gbk') + dataset=data.fillna(-1).values + return dataset +def get_batch(data,percent): + num_elements = np.uint32(percent * data.shape[0] / 100) + shuffled_data = data + np.random.shuffle(data) + return data[shuffled_data[:num_elements]] + +with tf.Session() as sess: + + init.run() + k=gConfig['training_epochs'] + data_set=read_data(gConfig['input_file']) + while k>0 : + n_batches = int(100/gConfig['percent'])+1 + for iteration in range(n_batches): + X_batch = data_set + print (X_batch) + #get_batch(data_set,gConfig['percent']) + #sess.run(training_op, feed_dict={X: X_batch}) + loss_val, reconstruction_loss_val, latent_loss_val = sess.run([loss, reconstruction_loss, latent_loss], feed_dict={X: X_batch}) + print("\r{}".format(k), "Train total cost:", loss_val, "\tReconstruction loss:", reconstruction_loss_val, "\tLatent loss:", latent_loss_val) + saver.save(sess, "./my_model_variational_variant.ckpt") + k=k-1 + #codings = hidden3 + # saver.restore(sess,"./my_model_variational.ckpt") + #codings_eval = codings.eval(feed_dict={X:data_set}) + + diff --git a/lessonThree/Anti-Fraud-App/checkpoint b/lessonThree/Anti-Fraud-App/checkpoint new file mode 100644 index 0000000..0d0532c --- /dev/null +++ b/lessonThree/Anti-Fraud-App/checkpoint @@ -0,0 +1,2 @@ +model_checkpoint_path: "my_model_variational_variant.ckpt" +all_model_checkpoint_paths: "my_model_variational_variant.ckpt" diff --git a/lessonThree/Anti-Fraud-App/config.ini b/lessonThree/Anti-Fraud-App/config.ini index e69de29..3de6ad6 100644 --- a/lessonThree/Anti-Fraud-App/config.ini +++ b/lessonThree/Anti-Fraud-App/config.ini @@ -0,0 +1,23 @@ + +[ints] +#聚类的数量 +k_num=10 +#聚类最大训练的步数 +num_epochs=10 +#聚类训练步数 +steps=10 +#VAE训练步数 +vae_steps=300 +seqlen=12 +[floats] +learning_rate = 0.01 +[strings] +mode = train +working_directory=working_directory/ +input_file=train_data/train_data.csv +model_path=kmeansMode/1542127988 + + + + + diff --git a/lessonThree/Anti-Fraud-App/data_util.py b/lessonThree/Anti-Fraud-App/data_util.py deleted file mode 100644 index 969d84b..0000000 --- a/lessonThree/Anti-Fraud-App/data_util.py +++ /dev/null @@ -1,4 +0,0 @@ -import numpy as np -import pandas as pd -import os - diff --git a/lessonThree/Anti-Fraud-App/execute.py b/lessonThree/Anti-Fraud-App/execute.py index e69de29..249e568 100644 --- a/lessonThree/Anti-Fraud-App/execute.py +++ b/lessonThree/Anti-Fraud-App/execute.py @@ -0,0 +1,50 @@ +import tensorflow as tf +import kmeansModel +import pandas as pd +import numpy as np +import getConfig +import sys +gConfig={} +gConfig=getConfig.get_config(config_file='config.ini') +def read_data(source_file): + data=pd.read_csv(source_file,encoding='utf-8') + dataset=data.values + return dataset + +def train(): + +#设置GPU管理的配置 + config = tf.ConfigProto() + config.gpu_options.allocator_type = 'BFC' + +#读取数据 + dataarray=read_data(gConfig['input_file']) + + + print("Size of data : ",dataarray.shape) +#在会话下进行训练 + with tf.Session(config=config) as sess: + + model_path=kmeansModel.trainAndSaveModel(dataarray,gConfig['steps']) + +def predicts(predict_set): + model_path=gConfig['model_path'] + predict=kmeansModel.predict(model_path,predict_set) + return predict + +if __name__=='__main__': + + if len(sys.argv) - 1: + gConfig = getConfig(sys.argv[1]) + else: + # get configuration from config.ini + gConfig = getConfig.get_config() + if gConfig['mode']=='train': + train() + elif gConfig['mode']=='server': + print('Sever Usage:python3 app.py') + + + + + diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126276/saved_model.pb b/lessonThree/Anti-Fraud-App/kmeansMode/1542126276/saved_model.pb new file mode 100644 index 0000000..ef312d9 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126276/saved_model.pb differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126276/variables/variables.data-00000-of-00001 b/lessonThree/Anti-Fraud-App/kmeansMode/1542126276/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000..07c8a44 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126276/variables/variables.data-00000-of-00001 differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126276/variables/variables.index b/lessonThree/Anti-Fraud-App/kmeansMode/1542126276/variables/variables.index new file mode 100644 index 0000000..1398e12 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126276/variables/variables.index differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126368/saved_model.pb b/lessonThree/Anti-Fraud-App/kmeansMode/1542126368/saved_model.pb new file mode 100644 index 0000000..087afc4 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126368/saved_model.pb differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126368/variables/variables.data-00000-of-00001 b/lessonThree/Anti-Fraud-App/kmeansMode/1542126368/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000..07c8a44 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126368/variables/variables.data-00000-of-00001 differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126368/variables/variables.index b/lessonThree/Anti-Fraud-App/kmeansMode/1542126368/variables/variables.index new file mode 100644 index 0000000..1398e12 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126368/variables/variables.index differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126570/saved_model.pb b/lessonThree/Anti-Fraud-App/kmeansMode/1542126570/saved_model.pb new file mode 100644 index 0000000..b07b18e Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126570/saved_model.pb differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126570/variables/variables.data-00000-of-00001 b/lessonThree/Anti-Fraud-App/kmeansMode/1542126570/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000..07c8a44 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126570/variables/variables.data-00000-of-00001 differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126570/variables/variables.index b/lessonThree/Anti-Fraud-App/kmeansMode/1542126570/variables/variables.index new file mode 100644 index 0000000..1398e12 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126570/variables/variables.index differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126868/saved_model.pb b/lessonThree/Anti-Fraud-App/kmeansMode/1542126868/saved_model.pb new file mode 100644 index 0000000..8028576 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126868/saved_model.pb differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126868/variables/variables.data-00000-of-00001 b/lessonThree/Anti-Fraud-App/kmeansMode/1542126868/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000..07c8a44 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126868/variables/variables.data-00000-of-00001 differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542126868/variables/variables.index b/lessonThree/Anti-Fraud-App/kmeansMode/1542126868/variables/variables.index new file mode 100644 index 0000000..1398e12 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542126868/variables/variables.index differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542127988/saved_model.pb b/lessonThree/Anti-Fraud-App/kmeansMode/1542127988/saved_model.pb new file mode 100644 index 0000000..69f9879 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542127988/saved_model.pb differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542127988/variables/variables.data-00000-of-00001 b/lessonThree/Anti-Fraud-App/kmeansMode/1542127988/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000..07c8a44 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542127988/variables/variables.data-00000-of-00001 differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542127988/variables/variables.index b/lessonThree/Anti-Fraud-App/kmeansMode/1542127988/variables/variables.index new file mode 100644 index 0000000..1398e12 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542127988/variables/variables.index differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542128836/saved_model.pb b/lessonThree/Anti-Fraud-App/kmeansMode/1542128836/saved_model.pb new file mode 100644 index 0000000..5eabb70 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542128836/saved_model.pb differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542128836/variables/variables.data-00000-of-00001 b/lessonThree/Anti-Fraud-App/kmeansMode/1542128836/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000..07c8a44 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542128836/variables/variables.data-00000-of-00001 differ diff --git a/lessonThree/Anti-Fraud-App/kmeansMode/1542128836/variables/variables.index b/lessonThree/Anti-Fraud-App/kmeansMode/1542128836/variables/variables.index new file mode 100644 index 0000000..1398e12 Binary files /dev/null and b/lessonThree/Anti-Fraud-App/kmeansMode/1542128836/variables/variables.index differ diff --git a/lessonThree/Anti-Fraud-App/kmeansModel.py b/lessonThree/Anti-Fraud-App/kmeansModel.py index 1e8961d..2d586bc 100644 --- a/lessonThree/Anti-Fraud-App/kmeansModel.py +++ b/lessonThree/Anti-Fraud-App/kmeansModel.py @@ -1 +1,115 @@ -import tensorflow as tf \ No newline at end of file +import tensorflow as tf +import numpy as np +from six.moves import xrange +import pandas as pd +import getConfig + +""" +代码结构： + +因为本次代码不涉及到复杂的神经网络的搭建和模型的训练，因为就不创建kmeansModel类了，直接定义所需的函数即可。 +一共三个函数： +serving_input_receiver_fn():用来获取模型参数，以便于进行模型保存时使用 + +trainAndSaveModel(input_set,steps)：用来训练模型并将模型保存到指定的文件夹下 + +predict(export_dir,predict_set)：重新加载模型，并对需要聚类数据进行聚类，返回数据所属类名。 + + +知识点： + +tf.estimator 是用来对模型进行训练和评估，包括模型训练，模型保存，模型加载等方法。大家如果感兴趣可以学习一下这个api。 + +tf.FixedLenFeature 返回的是一个定长的tensor，同时还有一个方法可以返回不定长的,那就是tensortf.FixedLenFeature。 + +tf.parse_example：是用来将模型解析tensor字典， +比如在features = tf.parse_example(model_placeholder, feature_spec)就是将feature_spec 解析成receiver_tensors张量字典然后返回。 + + +tf.estimator.export.ServingInputReceiver对象是将生成的特征Tensor和占位符组合在一起。 +tf.convert_to_tensor：将数组转成Tensor +tf.train.limit_epochs:设置epochs的数量 + +tf.contrib.factorization.KMeansClustering：这个是Kmeans聚类的类，涵盖了train,predict,cluster_centers，export_savedmodel等方法。有一点要注意，就是其输入的input_fn是一个function，而不是具体的数据。 + +""" + +def serving_input_receiver_fn(): + feature_spec = {"x": tf.FixedLenFeature(dtype=tf.float32, shape=[12])} + model_placeholder = tf.placeholder(dtype=tf.string,shape=[None],name='input') + receiver_tensors = {"model_inputs": model_placeholder} + features = tf.parse_example(model_placeholder, feature_spec) + return tf.estimator.export.ServingInputReceiver(features, receiver_tensors) + +def trainAndSaveModel(input_set,steps): + + gConfig={} + gConfig=getConfig.get_config(config_file='config.ini') + + input_fn = lambda: tf.train.limit_epochs(tf.convert_to_tensor(input_set, dtype=tf.float32), num_epochs=gConfig['num_epochs']) + kmeans = tf.contrib.factorization.KMeansClustering(num_clusters=gConfig['k_num'], use_mini_batch=False) + previous_centers = None + for _ in xrange(gConfig['steps']): + kmeans.train(input_fn)#调用train对训练数据进行训练 + centers = kmeans.cluster_centers()#保存质心 + if previous_centers is not None: + print ('delta:', centers - previous_centers) + previous_centers = centers + print ('score:', kmeans.score(input_fn)) + #将模型保存下来 + modelPath = kmeans.export_savedmodel(export_dir_base="kmeansMode/",serving_input_receiver_fn=serving_input_receiver_fn) + + print("训练完成,model文件存放在：") + print(modelPath) + + +""" +知识点： +TFRecords其实是一种二进制文件，虽然它不如其他格式好理解，但是它能更好的利用内存，更方便复制和移动，并且不需要单独的标签文件。 +包括前面我们讲到的tf.parse_example以及tf.train.Feature、tf.train.Example都是对这个二进制文件的操作。 +一般来说一个Example中包含Features，Features里包含Feature的字典，Feature里包含有一个 FloatList，也可以是ByteList或者Int64List。 + +""" + +def predict(export_dir,predict_set): + sess = tf.Session() + import tensorflow.contrib.factorization + #加载模型 + tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], export_dir) + from tensorflow.contrib import predictor + #predictor.from_saved_model是从模型来构造一个预测函数，可以读取之前保存的model，并对输入数据进行聚类。 + predict_fn = predictor.from_saved_model(export_dir) + inputList=[] + for test_data in predict_set: + #以下这块是关于TFRecords的操作，如果大家一时半会理解不了，就可以简单理解为是在准备预测数据特征。 + predictor_input_feature = { + 'x': tf.train.Feature( + float_list=tf.train.FloatList( + value=test_data + ) + ) + } + + input_for_predictor = tf.train.Example( + features=tf.train.Features( + feature=predictor_input_feature + ) + ) + #把输入数据转换为String + serialized_input = input_for_predictor.SerializeToString() + inputList.append(serialized_input) + + results = predict_fn({"model_inputs": inputList}) + clusterIndices = results['output'] + + #enumerate() 函数用于将一个可遍历的数据对象(如列表、元组或字符串)组合为一个索引序列，同时列出数据和数据下标。 + + for i, point in enumerate(predict_set): + clusterIndex = clusterIndices[i] + print ('point:', point, 'is in cluster', clusterIndex) + return clusterIndex + + + + + diff --git a/lessonThree/Anti-Fraud-App/sampled_data.csv b/lessonThree/Anti-Fraud-App/sampled_data.csv new file mode 100644 index 0000000..76635d8 --- /dev/null +++ b/lessonThree/Anti-Fraud-App/sampled_data.csv @@ -0,0 +1,10 @@ +,0,1,2,3,4,5,6,7 +0,-0.6319965,0.9363623,0.11962339,-3.9053032,-1.4392189,0.9460738,-0.04780117,1.2399702 +1,-0.0033503459,0.71606433,1.5091283,0.17764206,-1.3862783,-1.4228044,-0.3331036,-0.95360667 +2,-0.059529703,-1.2207987,-1.1939543,2.2094343,0.68686384,0.7169615,-0.7641363,0.054391183 +3,0.34732014,-0.06719242,-0.79393804,0.27349108,0.71757454,-0.7703055,0.54542124,-1.6733289 +4,-0.4911054,1.2711241,0.92903113,-0.75384694,0.6846527,-1.7292792,-0.33992317,-0.37706304 +5,-0.5575336,1.3310628,-0.4464654,-0.44233623,0.14736572,1.2980003,0.749322,1.0556818 +6,1.4449768,-0.69042015,-2.2388296,-1.0771358,-0.3423605,-0.4855051,1.4518228,-1.973025 +7,0.75124955,1.1796774,1.0435824,0.012941368,-0.78499895,0.5763671,-0.2909618,0.2336424 +8,0.35815558,-0.81797975,-3.3524156,1.3954812,-1.3640523,-0.034437098,-1.3486669,0.18941101 diff --git a/lessonThree/Anti-Fraud-App/train_data/train_data.csv b/lessonThree/Anti-Fraud-App/train_data/train_data.csv new file mode 100644 index 0000000..697ee1c --- /dev/null +++ b/lessonThree/Anti-Fraud-App/train_data/train_data.csv @@ -0,0 +1,10 @@ +77,90,6,57,97,25,90,60,9,25,27,28 +33,26,67,67,17,6,55,79,77,29,72,80 +33,49,95,60,47,66,96,61,14,1,79,11 +12,1,13,21,61,12,82,98,78,41,4,11 +51,30,39,83,12,83,73,31,46,45,6,90 +49,17,50,77,96,16,73,99,31,97,63,60 +14,40,69,7,1,11,89,85,30,11,34,9 +84,66,20,96,37,15,78,16,17,6,23,90 +21,28,34,36,70,8,94,68,17,61,21,8 +38,83,9,100,15,8,48,61,26,52,30,100 \ No newline at end of file diff --git a/lessonTwo/chinese_seq2seq_chatbot/__pycache__/getConfig.cpython-36.pyc b/lessonTwo/chinese_seq2seq_chatbot/__pycache__/getConfig.cpython-36.pyc new file mode 100644 index 0000000..78bf7ad Binary files /dev/null and b/lessonTwo/chinese_seq2seq_chatbot/__pycache__/getConfig.cpython-36.pyc differ diff --git a/lessonTwo/chinese_seq2seq_chatbot/data_utls.py b/lessonTwo/chinese_seq2seq_chatbot/data_utls.py index 1a108da..1a6f026 100644 --- a/lessonTwo/chinese_seq2seq_chatbot/data_utls.py +++ b/lessonTwo/chinese_seq2seq_chatbot/data_utls.py @@ -22,7 +22,7 @@ #c、找出我们想要的数据存储下来 #知识点：open函数 for循环结构、数据类型（list的操作）、continue convs = [] # 用于存储对话的列表 -with open(conv_path,encoding='ISO-8859-1') as f: +with open(conv_path,encoding='utf-8') as f: one_conv = [] # 存储一次完整对话 for line in f: line = line.strip('\n').replace('/', '')#去除换行符，并将原文件中已经分词的标记去掉，重新用结巴分词. diff --git a/lessonTwo/chinese_seq2seq_chatbot/prepareData.py b/lessonTwo/chinese_seq2seq_chatbot/prepareData.py index f8a8eac..fa00f5f 100644 --- a/lessonTwo/chinese_seq2seq_chatbot/prepareData.py +++ b/lessonTwo/chinese_seq2seq_chatbot/prepareData.py @@ -97,32 +97,4 @@ def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_ return (enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path) -# 用于语句切割的正则表达 -#_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])") -#_DIGIT_RE = re.compile(br"\d") -#def basic_tokenizer(sentence): - #将一个语句中的字符切割成一个list，这样是为了下一步进行向量化训练 - # words = [] - # for space_separated_fragment in sentence.strip().split(): - # words.extend(re.split(_WORD_SPLIT, space_separated_fragment)) - # return [w for w in words if w] - -#def sentence_to_token_ids(sentence, vocabulary, normalize_digits=True):#将输入语句从中文字符转换成数字符号 - - # words = basic_tokenizer(sentence) - # if not normalize_digits: - # return [vocabulary.get(w, UNK_ID) for w in words] - # # Normalize digits by 0 before looking words up in the vocabulary. - # return [vocabulary.get(re.sub(_DIGIT_RE, b"0", w), UNK_ID) for w in words] - -#def initialize_vocabulary(vocabulary_path):#初始化字典，这里的操作与上面的48行的的作用是一样的，是对调字典中的key-value - # if gfile.Exists(vocabulary_path): - # rev_vocab = [] - # with open(vocabulary_path, "r") as f: - # rev_vocab.extend(f.readlines()) - # rev_vocab = [line.strip() for line in rev_vocab] - # vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)]) - # return vocab, rev_vocab - #else: - # raise ValueError("Vocabulary file %s not found.", vocabulary_path)