diff --git a/pyengine/engine/controller/vector_engine.py b/pyengine/engine/controller/vector_engine.py index 7736914782..a8d081c622 100644 --- a/pyengine/engine/controller/vector_engine.py +++ b/pyengine/engine/controller/vector_engine.py @@ -108,7 +108,8 @@ class VectorEngine(object): FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1, 'type': 'index', 'filename': index_filename}) - pass + db.session.commit() + VectorEngine.group_dict = None else: # we still can insert into exist raw file, update database @@ -152,9 +153,9 @@ class VectorEngine(object): vectors.append(vector) result = scheduler_instance.search(index_map, vectors, limit) - vector_id = 0 + # vector_id = 0 - return VectorEngine.SUCCESS_CODE, vector_id + return VectorEngine.SUCCESS_CODE, result @staticmethod @@ -184,7 +185,8 @@ class VectorEngine(object): VectorEngine.group_dict[group_id].append(vector) - print('InsertVectorIntoRawFile: ', VectorEngine.group_dict[group_id]) + # print('InsertVectorIntoRawFile: ', VectorEngine.group_dict[group_id]) + print("cache size: ", len(VectorEngine.group_dict[group_id])) return filename diff --git a/pyengine/engine/controller/views.py b/pyengine/engine/controller/views.py index 425b337d3c..4629688270 100644 --- a/pyengine/engine/controller/views.py +++ b/pyengine/engine/controller/views.py @@ -16,6 +16,7 @@ class Vector(Resource): self.__parser.add_argument('vector', type=float, action='append', location=['json']) def post(self, group_id): + print(request.json) args = self.__parser.parse_args() vector = args['vector'] code = VectorEngine.AddVector(group_id, vector) diff --git a/pyengine/engine/settings.py b/pyengine/engine/settings.py index a73af880b2..428deffc19 100644 --- a/pyengine/engine/settings.py +++ b/pyengine/engine/settings.py @@ -5,5 +5,5 @@ SQLALCHEMY_TRACK_MODIFICATIONS = False # SECRET_KEY='A0Zr98j/3yX R~XHH!jmN]LWX/,?RT' SQLALCHEMY_DATABASE_URI = "mysql+pymysql://vecwise@127.0.0.1:3306/vecdata" -ROW_LIMIT = 10000000 +ROW_LIMIT = 1000000 DATABASE_DIRECTORY = '/tmp' \ No newline at end of file diff --git a/pyengine/tests/test_function.py b/pyengine/tests/test_function.py new file mode 100644 index 0000000000..af90871750 --- /dev/null +++ b/pyengine/tests/test_function.py @@ -0,0 +1,53 @@ +import unittest +import numpy as np +import requests +import logging +import json + +url = "http://127.0.0.1:5000" + + +# TODO: LOG and Assert +class TestEngineFunction(unittest.TestCase): + def test_1m_add(self): + d = 4 + nb = 120 + nq = 1 + k = 10 + _, xb, xq = get_dataset(d, nb, 1, nq) + + groupid = "5m" + + # route_group = url + "/vector/group/" + groupid + # r = requests.post(route_group, json={"dimension": d}) + # + # # import dataset + # vector_add_route = url + "/vector/add/" + groupid + # for i in xb: + # data = dict() + # data['vector'] = i.tolist() + # r = requests.post(vector_add_route, json=data) + + # search dataset + vector_search_route = url + "/vector/search/" + groupid + data = dict() + data['vector'] = xq.tolist() + data['limit'] = k + r = requests.get(vector_search_route, json=data) + + print("finish") + +def get_dataset(d, nb, nt, nq): + d1 = 10 # intrinsic dimension (more or less) + n = nb + nt + nq + rs = np.random.RandomState(1338) + x = rs.normal(size=(n, d1)) + x = np.dot(x, rs.rand(d1, d)) + x = x * (rs.rand(d) * 4 + 0.1) + x = np.sin(x) + x = x.astype('float32') + return x[:nt], x[nt:-nq], x[-nq:] + + +if __name__ == "__main__": + unittest.main()