diff --git a/apps/Tests/__init__.py b/apps/Tests/__init__.py new file mode 100644 index 00000000..507d7646 --- /dev/null +++ b/apps/Tests/__init__.py @@ -0,0 +1 @@ +from .myApp import * diff --git a/apps/Tests/algs/Algs.yaml b/apps/Tests/algs/Algs.yaml new file mode 100644 index 00000000..b40b9334 --- /dev/null +++ b/apps/Tests/algs/Algs.yaml @@ -0,0 +1,24 @@ +initExp: + args: + dummy: + type: bool + description: This is here because there's a bug that requires myAlg.initExp to have at least one argument. + default: true + rets: + type: bool + values: true + +getQuery: + rets: + type: bool + values: true + +processAnswer: + rets: + type: bool + values: true + +getModel: + rets: + type: bool + values: true diff --git a/apps/Tests/algs/TestAlg.py b/apps/Tests/algs/TestAlg.py new file mode 100644 index 00000000..bd5627ad --- /dev/null +++ b/apps/Tests/algs/TestAlg.py @@ -0,0 +1,25 @@ + +class MyAlg: + def initExp(self, butler, dummy): + + butler.algorithms.set(key='algorithms_foo', value='algorithms_bar') + + return True + + def getQuery(self, butler): + + assert butler.experiment.get(key='experiment_foo') == 'experiment_bar' + assert butler.algorithms.get(key='algorithms_foo') == 'algorithms_bar' + + return True + + def processAnswer(self, butler): + + assert butler.experiment.get(key='experiment_foo') == 'experiment_bar' + assert butler.algorithms.get(key='algorithms_foo') == 'algorithms_bar' + + return True + + def getModel(self, butler): + + return True diff --git a/apps/Tests/algs/__init__.py b/apps/Tests/algs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/Tests/dashboard/Dashboard.py b/apps/Tests/dashboard/Dashboard.py new file mode 100644 index 00000000..56f6f2d4 --- /dev/null +++ b/apps/Tests/dashboard/Dashboard.py @@ -0,0 +1,5 @@ +from next.apps.AppDashboard import AppDashboard + + +class MyAppDashboard(AppDashboard): + pass diff --git a/apps/Tests/dashboard/__init__.py b/apps/Tests/dashboard/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/Tests/dashboard/myAppDashboard.html b/apps/Tests/dashboard/myAppDashboard.html new file mode 100644 index 00000000..326a9898 --- /dev/null +++ b/apps/Tests/dashboard/myAppDashboard.html @@ -0,0 +1,2 @@ + +{% extends "basic.html" %} diff --git a/apps/Tests/myApp.py b/apps/Tests/myApp.py new file mode 100644 index 00000000..73f86f0b --- /dev/null +++ b/apps/Tests/myApp.py @@ -0,0 +1,38 @@ +import json +import next.utils as utils +import next.apps.SimpleTargetManager + +class MyApp: + def __init__(self,db): + self.app_id = 'Tests' + self.TargetManager = next.apps.SimpleTargetManager.SimpleTargetManager(db) + + def initExp(self, butler, init_algs, args): + + butler.experiment.set(key='experiment_foo', value='experiment_bar') + + init_algs({}) + + return args + + def getQuery(self, butler, alg, args): + + assert butler.experiment.get(key='experiment_foo') == 'experiment_bar' + + assert alg() + + return {} + + def processAnswer(self, butler, alg, args): + + assert alg({}) + + assert butler.experiment.get(key='experiment_foo') == 'experiment_bar' + + return {} + + def getModel(self, butler, alg, args): + + assert alg() + + return True diff --git a/apps/Tests/myApp.yaml b/apps/Tests/myApp.yaml new file mode 100644 index 00000000..5afdd5c7 --- /dev/null +++ b/apps/Tests/myApp.yaml @@ -0,0 +1,12 @@ +extends: [base.yaml] +initExp: + args: + app_id: + values: [Tests] + args: + values: + alg_list: + values: + values: + alg_id: + values: [TestAlg] diff --git a/apps/Tests/tests/__init__.py b/apps/Tests/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/Tests/tests/test_api.py b/apps/Tests/tests/test_api.py new file mode 100644 index 00000000..9f5ef92b --- /dev/null +++ b/apps/Tests/tests/test_api.py @@ -0,0 +1,120 @@ +import numpy +import numpy.random +import random +import json +import time +import requests +from scipy.linalg import norm +from multiprocessing import Pool +import os +import sys +try: + import next.apps.test_utils as test_utils +except: + file_dir = '/'.join(__file__.split('/')[:-1]) + sys.path.append('{}/../../../next/apps'.format(file_dir)) + import test_utils + +app_id = 'Tests' + + +def test_api(assert_200=True, num_objects=5, desired_dimension=2, + total_pulls_per_client=4, num_experiments=1, num_clients=6): + + pool = Pool(processes=num_clients) + supported_alg_ids = ['TestAlg'] + alg_list = [] + for idx, alg_id in enumerate(supported_alg_ids): + alg_item = {} + alg_item['alg_id'] = alg_id + alg_item['alg_label'] = alg_id + alg_list.append(alg_item) + + params = [] + for algorithm in alg_list: + params.append({'alg_label': algorithm['alg_label'], + 'proportion': 1./len(alg_list)}) + algorithm_management_settings = {} + algorithm_management_settings['mode'] = 'fixed_proportions' + algorithm_management_settings['params'] = params + + # Test POST Experiment + initExp_args_dict = {} + initExp_args_dict['app_id'] = 'Tests' + initExp_args_dict['args'] = {} + initExp_args_dict['args']['participant_to_algorithm_management'] = 'one_to_many' + initExp_args_dict['args']['algorithm_management_settings'] = algorithm_management_settings + initExp_args_dict['args']['alg_list'] = alg_list + initExp_args_dict['args']['instructions'] = 'You want instructions, here are your test instructions' + initExp_args_dict['args']['debrief'] = 'You want a debrief, here is your test debrief' + initExp_args_dict['args']['targets'] = {} + initExp_args_dict['args']['targets']['n'] = num_objects + + exp_info = [] + for ell in range(num_experiments): + initExp_response_dict, exp_uid = test_utils.initExp(initExp_args_dict) + exp_info += [exp_uid] + + # Generate participants + participants = [] + pool_args = [] + for i in range(num_clients): + participant_uid = '%030x' % random.randrange(16**30) + participants.append(participant_uid) + + experiment = numpy.random.choice(exp_info) + exp_uid = experiment['exp_uid'] + pool_args.append((exp_uid, participant_uid, total_pulls_per_client, assert_200)) + results = pool.map(simulate_one_client, pool_args) + + for result in results: + print result + + test_utils.getModel(exp_uid, app_id, supported_alg_ids, alg_list) + + +def simulate_one_client(input_args): + + exp_uid, participant_uid, total_pulls, assert_200 = input_args + + getQuery_times = [] + processAnswer_times = [] + + for t in range(total_pulls): + print "Participant {1} has taken {0} pulls".format(t, participant_uid) + # test POST getQuery # + widget = random.choice([True] + 4*[False]) + widget = True + getQuery_args_dict = {'args': {'participant_uid': participant_uid, + 'widget': widget}, + 'exp_uid': exp_uid} + + query_dict, dt = test_utils.getQuery(getQuery_args_dict) + getQuery_times += [dt] + + if widget: + query_dict = query_dict['args'] + query_uid = query_dict['query_uid'] + + ts = test_utils.response_delay() + # sleep for a bit to simulate response time + + response_time = time.time() - ts + + # test POST processAnswer + processAnswer_args_dict = {} + processAnswer_args_dict["exp_uid"] = exp_uid + processAnswer_args_dict["args"] = {} + processAnswer_args_dict["args"]["query_uid"] = query_uid + processAnswer_args_dict["args"]['response_time'] = response_time + + processAnswer_json_response, dt = test_utils.processAnswer(processAnswer_args_dict) + processAnswer_times.append(dt) + + r = test_utils.format_times(getQuery_times, processAnswer_times, total_pulls, + participant_uid) + + return r + +if __name__ == '__main__': + test_api() diff --git a/apps/Tests/widgets/getQuery_widget.html b/apps/Tests/widgets/getQuery_widget.html new file mode 100644 index 00000000..e69de29b diff --git a/next/apps/App.py b/next/apps/App.py index a68a8623..9d9ccadd 100644 --- a/next/apps/App.py +++ b/next/apps/App.py @@ -102,9 +102,10 @@ def initExp(self, exp_uid, args_json): args_dict['start_date'] = utils.datetime2str(utils.datetimeNow()) self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())}) utils.debug_print("ASD "+str(args_dict)) + self.butler.experiment.set(value={'exp_uid': exp_uid}) args_dict['args'] = self.init_app(exp_uid, args_dict['args']['alg_list'], args_dict['args']) args_dict['git_hash'] = git_hash - self.butler.experiment.set(value=args_dict) + self.butler.experiment.set_many(key_value_dict=args_dict) return '{}', True, '' except Exception, error: exc_type, exc_value, exc_traceback = sys.exc_info()