five

lessonFive/app.py

@@ -0,0 +1,47 @@
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+import os
+from flask import Flask, render_template, request, make_response
+from flask import jsonify
+from flask import Flask
+from flask import jsonify
+from flask import Response
+import sys
+import time  
+import hashlib
+import threading
+import execute
+import pandas as pd
+
+app = Flask(__name__) 
+#路由注解，我们这里使用的是path的形式进行传参
+#
+@app.route('/predict/<a>/<b>/<c>/<d>/<e>/<f>/<g>/<h>/<i>/<j>/<k>/<l>/<m>/<n>/<o>/<p>/<q>/<r>', methods=['GET'])
+def predict(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r):
+	#获取url传来的需要进行预测的数据
+	line=[a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r]
+	lines=range(19)
+	lines=[i for i in line]
+	#因为我们全量的数据是19列，所以我们要在数据后面增加一个元素
+	lines.append(0)
+	COLUMNS = ['1','2','3', '4',  '5',  '6',  '7',  '8',  '9',  '10', '11', '12', '13', '14', '15', '16', '29', '30', '31']
+
+	#将数组转换成dataframe[[1,2,3]]
+
+	lines= pd.DataFrame([lines],columns=COLUMNS)
+
+	#predict_result=train.predict(lines)
+	predict_result=execute.predict(sess,lines,model)
+
+	#返回
+	return jsonify( { 'result of cluster': str(predict_result) } )
+
+#初始化session，大家想一下如果不初始化会有什么问题？
+sess = tf.Session()
+sess, model = execute.init_session(sess, conf='config.ini')
+
+if (__name__ == "__main__"): 
+    app.run(host = '0.0.0.0', port = 8088) 
+
+

lessonFive/config.ini

@@ -0,0 +1,19 @@
+[strings]
+# Mode : train, test, serve
+mode = train
+resource_data=data/source_data.csv
+
+training_set=data/train_data.csv
+
+test_set=data/test_data.csv
+
+model_dir=model
+
+[ints]
+
+
+[floats]
+learning_rate = 0.01
+keeps=0.5
+end_loss=0.1
+

lessonFive/data_ults.py

@@ -0,0 +1,73 @@
+# coding=utf-8
+
+import pandas as pd
+import numpy as np
+import getConfig
+
+import os
+import random
+#常规的获取配置信息
+gConfig = {}
+
+gConfig=getConfig.get_config()
+
+conv_path = gConfig['resource_data']
+
+#定义csv的列名，用于区分特征列和标示列
+
+COLUMNS = ['1','2',  '3',  '4',  '5',  '6',  '7',  '8',  '9',  '10', '11', '12', '13', '14', '15', '16', '29', '30', '31']
+
+resource_data=pd.read_csv(conv_path,skipinitialspace=True,skiprows=1, names=COLUMNS,low_memory=False)
+#利用pd 的dataframe特性进行缺省值补充
+"""
+知识点：
+缺省值补全：
+df.fillna(0)
+df.fillna('missing')
+df.fillna(method='pad')
+
+df.fillna(method='bfill',limit=1)
+
+df.fillna(df.mean()）
+
+dataframe.to_csv
+
+
+"""
+resource_data=resource_data.fillna(method='bfill')
+
+def sample_test_data(resource_data,TESTSET_SIZE):
+
+	    test_index = random.sample([i for i in range(len(resource_data))],TESTSET_SIZE)
+
+	    train_data=[]
+	    test_data=[]
+
+	    for i in range(len(resource_data)):
+	        if i in test_index:
+	          #print(resource_data.loc[i])
+	          test_data.append(resource_data.loc[i])
+
+	        else:
+	           train_data.append(resource_data.loc[i])
+	        if i % 1000 == 0:
+	           print(len(range(len(resource_data))), '处理进度：', i)
+	    train_data_tocsv=pd.DataFrame(train_data)
+
+	    test_data_tocsv=pd.DataFrame(test_data)
+
+	    train_data_path=gConfig['training_set']
+
+	    test_data_path=gConfig['test_set']
+
+	    train_data_tocsv.to_csv(train_data_path)
+
+	    test_data_tocsv.to_csv(test_data_path)
+
+
+sample_test_data(resource_data,10000)
+
+
+
+
+

lessonFive/dnnClassifierModel.py

@@ -0,0 +1,20 @@
+import tensorflow as tf
+
+import getConfig
+
+gConfig={}
+
+gConfig=getConfig.get_config(config_file='config.ini')
+
+
+class dnnClassifierModel(object):
+
+	def __init__(self):
+
+
+		def createModel(feature_columns):
+			return tf.contrib.learn.DNNClassifier(
+				feature_columns=feature_cols,
+				hidden_units=[105, 105, 105, 105 ],
+				model_dir=gConfig['model_dir'])
+

lessonFive/execute.py

@@ -0,0 +1,100 @@
+# -*- coding:utf-8 -*-
+from time import strftime, localtime 
+from datetime import timedelta, date 
+import itertools
+import tensorflow as tf
+import shutil
+import os
+import pandas as pd
+import numpy as np
+import getConfig
+import sys
+
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
+#获取配置和数据特征列
+gConfig={}
+
+gConfig=getConfig.get_config(config_file='config.ini')
+
+COLUMNS = ['1','2',  '3',  '4',  '5',  '6',  '7',  '8',  '9',  '10', '11', '12', '13', '14', '15', '16', '29', '30', '31']
+
+FEATURES=['1','2',  '3',  '4',  '5',  '6',  '7',  '8',  '9',  '10', '11', '12', '13', '14', '15', '16', '29', '30']
+
+LABEL=['31']
+
+#定义GPU的内存管理算法
+
+config = tf.ConfigProto()
+config.gpu_options.allocator_type = 'BFC'
+
+#定义输入函数
+def input_fn(data_set):
+  feature_cols = {k: tf.constant(data_set[k].values) for k in FEATURES}
+  labels = tf.constant(data_set[LABEL].values)
+  return feature_cols, labels
+#定义训练函数
+def train():
+  # 利用pandas来读取CSV的数据，dataframe格式
+  training_set = pd.read_csv(gConfig['training_set'], skipinitialspace=True,skiprows=1, names=COLUMNS)
+  test_set = pd.read_csv(gConfig['test_set'], skipinitialspace=True,skiprows=1, names=COLUMNS)
+
+  # 划出和定义特征列
+
+
+  """
+
+  tf.contrib.layers.real_valued_column：为连续的列元素设置一个实值列
+
+  tf.contrib.learn.DNNClassifier
+
+  classifier.fit
+
+  classifier.evaluate
+
+  """
+  feature_cols = [tf.contrib.layers.real_valued_column(k) for k in FEATURES]
+
+ # 构造一个4层，每层105个神经元的全连接的DNN计算图.
+  classifier = tf.contrib.learn.DNNClassifier(feature_columns=feature_cols,hidden_units=[105, 105, 105, 105 ],dropout=gConfig['keeps'],model_dir=gConfig['model_dir'])
+
+  loss_score=1
+  # 开始进行训练，知道满足条件后停止
+  while loss_score>gConfig['end_loss']:
+
+      classifier.fit(input_fn=lambda: input_fn(training_set),steps=100)
+      # 测试和评价模型的准确度
+      ev = classifier.evaluate(input_fn=lambda: input_fn(test_set), steps=1)
+      accuracy_score = ev["accuracy"]
+      print("模型准确率: {0:f}".format(accuracy_score))
+
+def init_session(sess,conf='config.ini'):
+    global gConfig
+    gConfig=getConfig.get_config(conf)
+    feature_cols = [tf.contrib.layers.real_valued_column(k) for k in FEATURES]
+    model=tf.contrib.learn.DNNClassifier(feature_columns=feature_cols,hidden_units=[105, 105, 105, 105 ],model_dir=gConfig['model_dir'])
+    return sess, model
+
+def predict(sess,predict_set,model):
+
+    y=model.predict(input_fn=lambda: input_fn(predict_set))
+
+    predictions = list(y)
+    return predictions
+
+if __name__ == "__main__":
+
+    if len(sys.argv) - 1:
+        gConfig = getConfig(sys.argv[1])
+    else:
+        # get configuration from config.ini
+        gConfig = getConfig.get_config()
+    if gConfig['mode']=='train':
+        train()
+    elif gConfig['mode']=='server':
+        print('Sever Usage:python3 app.py')
+
+
+
+
+

lessonFive/getConfig.py

@@ -0,0 +1,9 @@
+import configparser
+def get_config(config_file='config.ini'):
+    parser=configparser.ConfigParser()
+    parser.read(config_file)
+    # get the ints, floats and strings
+    _conf_ints = [(key, int(value)) for key, value in parser.items('ints')]
+    _conf_floats = [(key, float(value)) for key, value in parser.items('floats')]
+    _conf_strings = [(key, str(value)) for key, value in parser.items('strings')]
+    return dict(_conf_ints + _conf_floats + _conf_strings)

lessonOne/imgClassifierWeb/cnnModel.py

@@ -106,7 +106,7 @@ def fc_layer(flattened_layer, num_inputs, num_outputs):
             fc_resultl = tf.matmul(flattened_layer, fc_weights)
             return fc_resultl
         batch_size=gConfig['percent']*gConfig['dataset_size']/100
-        self.data_tensor=tf.placeholder(tf.float32,shape=[batch_size,gConfig['im_dim'], gConfig['im_dim'],gConfig['num_channels']],name='data_tensor')
+        self.data_tensor=tf.placeholder(tf.float32,shape=tf.placeholder(tf.float32,shape=[batch_size,gConfig['im_dim'], gConfig['im_dim'],gConfig['num_channels']],name='data_tensor'))
         self.label_tensor=tf.placeholder(tf.int32,shape=[batch_size],name='label_tensor')
         keep_prop=tf.Variable(initial_value=0.5,name="keep_prop")
         self.fc_result=create_CNN(input_data=self.data_tensor,num_classes=gConfig['num_dataset_classes'],keep_prop=gConfig['keeps'])

lessonTwo/chinese_seq2seq_chatbot/seq2seq_model.py

@@ -198,30 +198,17 @@ def seq2seq_f(encoder_inputs, decoder_inputs, do_decode):
                for i in xrange(len(self.decoder_inputs) - 1)]
 
 
-"""
-知识点”
-
-lambda表达式
-
-
-
-
-
-
-"""
-
-
 
     # Training outputs and losses.
     if forward_only:
-      self.outputs, self.losses = tf.contrib.legacy_seq2seq.model_with_buckets(
-          self.encoder_inputs, self.decoder_inputs, targets,
-          self.target_weights, buckets, lambda x, y: seq2seq_f(x, y, True),
-          softmax_loss_function=softmax_loss_function)
+        self.outputs, self.losses = tf.contrib.legacy_seq2seq.model_with_buckets(
+        self.encoder_inputs, self.decoder_inputs, targets,
+        self.target_weights, buckets, lambda x, y: seq2seq_f(x, y, True),
+        softmax_loss_function=softmax_loss_function)
       # If we use output projection, we need to project outputs for decoding.
-      if output_projection is not None:
-        for b in xrange(len(buckets)):
-          self.outputs[b] = [
+        if output_projection is not None:
+           for b in xrange(len(buckets)):
+             self.outputs[b] = [
               tf.matmul(output, output_projection[0]) + output_projection[1]
               for output in self.outputs[b]
           ]