redisDB.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455
  1. 'use strict';
  2. const C = require('../config');
  3. const F = require('../common/function');
  4. const path = require('path');
  5. const _ = require('underscore');
  6. _.str = require('underscore.string');
  7. _.v = require('validator');
  8. var co = require('co');
  9. module.exports = function (app, commonManager) {
  10. let modelMap = app.model_mgr.model_map;
  11. let managerMap = commonManager.mgr_map;
  12. let model_map = app.model_mgr.model_map;
  13. let mgr_map = commonManager.mgr_map;
  14. let that = this;
  15. // redis key 前缀 - hash
  16. this.key_prefix_hash = '{rk}_hash_%s_%s'; // table increment
  17. // redis key 前缀 - set
  18. this.key_prefix_set = '{rk}_set_%s_%s_%s'; // table field value
  19. // redis hash自增key
  20. this.key_prefix_has_inc = '{rk}_hash_inc';
  21. // redis 临时区间结果集自增key
  22. this.key_prefix_tmp_set_inc = '{rk}_tmp_set_inc';
  23. // redis sort set key 前缀
  24. this.key_prefix_sort_set = '{rk}_zset_%s_%s'; // table field
  25. // 数据类型 s:字符串 n:运算数
  26. this.type = {s:'string', n:'number'};
  27. this.getKeyHash = function (table, increment) {
  28. return _.str.vsprintf(this.key_prefix_hash, [table, increment]);
  29. }
  30. this.getKeySet = function (table, field, value) {
  31. return _.str.vsprintf(this.key_prefix_set, [table, field, value]);
  32. }
  33. this.getKeySortSet = function (table, field) {
  34. return _.str.vsprintf(this.key_prefix_sort_set, [table, field]);
  35. }
  36. /**
  37. * 获取最优过期时间
  38. * @param key
  39. * @param expire
  40. * @returns {int}
  41. */
  42. this.getBestExpire = function* (key, expire) {
  43. return null; // 关闭超时
  44. let key_expire = yield managerMap.redis.ttl(key);
  45. if(key_expire < expire) {
  46. key_expire = expire;
  47. }else if(expire == -1) {
  48. key_expire = expire;
  49. }
  50. return key_expire;
  51. }
  52. /**
  53. * 获取数组交集
  54. * @param arrays
  55. * @returns {array}
  56. */
  57. this.intersection = function (arrays) {
  58. if(arrays.length == 0) return new Array();
  59. if(arrays.length == 1) return arrays[0];
  60. // 获取长度最小数组的索引
  61. let min_index = -1;
  62. for (let i = 0; i < arrays.length; i++) {
  63. if(min_index == -1 || arrays[min_index].length > arrays[i].length) {
  64. min_index = i;
  65. }
  66. }
  67. let intersection_array = new Array();
  68. for (let i = 0; i < arrays[min_index].length; i++) {
  69. let is_in = true; // 是否交集 默认true
  70. for (let j = 0; j < arrays.length; j++) {
  71. if(j == min_index) continue;
  72. if(arrays[j].indexOf(arrays[min_index][i]) == -1) {
  73. is_in = false;
  74. break;
  75. }
  76. }
  77. if(is_in == true && intersection_array.indexOf(arrays[min_index][i]) == -1) {
  78. intersection_array.push(arrays[min_index][i]);
  79. }
  80. }
  81. return intersection_array;
  82. }
  83. /**
  84. * 判断是否为数字类型
  85. * @param str
  86. * @returns {bool}
  87. */
  88. this.checkIsNumber = function (str) {
  89. if(F.isNull(str)) return false;
  90. let number_map = '1234567890.'; // .为了支持小数
  91. let is_number = true;
  92. for (let i = 0; i < str.length; i++) {
  93. if(number_map.indexOf(str.charAt(i)) == -1) {
  94. is_number = false;
  95. break;
  96. }
  97. }
  98. return is_number;
  99. }
  100. this.incr = 0;
  101. /**
  102. * 查询结果集
  103. * @param table
  104. * @param option
  105. * option包括:
  106. * fields 字符串 例如:"name,age" 不能空
  107. * where 字符串 例如"name = ? and age > ?" 不能空 只支持 = > < >= <=
  108. * values 数组 例如['fdl', 2]
  109. * @returns {}
  110. */
  111. this.queryResultFromRedisDB = function* (table, option, isForQuery=false) {
  112. let key_array = []; // 等值操作符key集合
  113. let range_key_array = []; // 范围key集合
  114. // 解析where
  115. let where_array = option.where.split(' and ');
  116. // 多个查询条件使用inter
  117. for (let i = 0; i < where_array.length; i++) {
  118. let split_str;
  119. if(where_array[i].indexOf('>=') != -1) {
  120. split_str = '>=';
  121. }else if(where_array[i].indexOf('<=') != -1) {
  122. split_str = '<=';
  123. }else if(where_array[i].indexOf('>') != -1){
  124. split_str = '>';
  125. }else if(where_array[i].indexOf('<') != -1) {
  126. split_str = '<';
  127. }else if(where_array[i].indexOf('=') != -1) {
  128. split_str = '=';
  129. }else {
  130. F.throwErr('where sql err.');
  131. }
  132. let param_array = where_array[i].split(split_str);
  133. let field = _.str.trim(param_array[0], ' ');
  134. let value = _.str.trim(param_array[1], [' ', '"', '\'']);
  135. if(value == '?') {
  136. value = option.values.shift();
  137. }
  138. let option_min;
  139. let option_max;
  140. switch(split_str) {
  141. case '=':
  142. key_array.push(this.getKeySet(table, field, value));
  143. break;
  144. case '>':
  145. option_min = '(' + value;
  146. option_max = '+inf';
  147. break;
  148. case '<':
  149. option_min = '-inf';
  150. option_max = '(' + value;
  151. break;
  152. case '>=':
  153. option_min = value;
  154. option_max = '+inf';
  155. break;
  156. case '<=':
  157. option_min = '-inf';
  158. option_max = value;
  159. break;
  160. }
  161. if(split_str != '=') {
  162. let sort_set_key = this.getKeySortSet(table, field);
  163. range_key_array.push({"key":sort_set_key,"min":option_min,"max":option_max});
  164. }
  165. }
  166. let sort_field = "";
  167. let isdesc = "";
  168. let limit_first = "";
  169. let limit_second = "";
  170. if (isForQuery) {
  171. if (!F.isNull(option.order)) {
  172. let order_array = option.order.split(',');
  173. if (order_array.length > 1) F.throwErr("redis db can not contain multi order");
  174. order_array = option.order.split(/\s+/);
  175. sort_field = _.str.trim(order_array[0], ' ');
  176. isdesc = _.str.trim(order_array[1], ' ');
  177. }
  178. if (!F.isNull(option.limit)) {
  179. let limit_array = option.limit.split(',');
  180. if (limit_array.length > 2) F.throwErr("redis db limit format is wrong");
  181. limit_first = _.str.trim(limit_array[0], ' ');
  182. limit_second = _.str.trim(limit_second[1], ' ');
  183. }
  184. }
  185. let query_res = yield managerMap.redis.mysinter(table,key_array,range_key_array,sort_field,isdesc,limit_first,limit_second);
  186. return query_res;
  187. }
  188. /**
  189. * 批量插入
  190. * * datas 数据对象数组
  191. */
  192. this.batchInsert = function* (table, option) {
  193. if (F.isNull(option) || F.isNull(option.datas)) F.throwErr('params err.');
  194. let res = [];
  195. for (let i = 0; i < option.datas.length; i++) {
  196. option.data = option.datas[i];
  197. let id = yield that.insertRedisDB(table, option);
  198. res.push(id);
  199. }
  200. return res;
  201. }
  202. /**
  203. * 插入
  204. * @param table
  205. * @param option
  206. * option包括:
  207. * data 数据对象
  208. * expire 过期时间 -1为永不过期
  209. * index 数据描述对象 - 设置该值表示添加索引
  210. * field:type type取值: n:number, s:string
  211. * 例如 {name:s, age:n}
  212. * @returns {}
  213. */
  214. this.insertRedisDB = function* (table, option) {
  215. if(F.isNull(table) || F.isNull(option) || F.isNull(option.data) || F.isNull(option.expire) || F.isNull(option.index)) {
  216. F.throwErr('params err.');
  217. }
  218. // 判断option.index值是否正确
  219. delete option.index.auto_id; // 先删除之前的auto_id index
  220. let type_arr = F.values(that.type);
  221. for (let field in option.index) {
  222. if(type_arr.indexOf(option.index[field]) == -1) F.throwErr('index err.');
  223. if(option.index[field] == that.type.n) {
  224. if(option.data.hasOwnProperty(field) == false) F.throwErr(field+' index no contain.');
  225. if(typeof(option.data[field]) != 'number') F.throwErr(field+' index type err.');
  226. }
  227. }
  228. // 添加 hash映射 set映射
  229. let key_hash_inc = 0;
  230. if (!F.isNull(option.data.auto_id)) key_hash_inc = option.data.auto_id;
  231. else key_hash_inc = yield managerMap.redis.hincrby(this.key_prefix_has_inc, table);
  232. let key_hash = this.getKeyHash(table, key_hash_inc);
  233. option.data.auto_id = key_hash_inc;
  234. option.index.auto_id = this.type.n;
  235. let fields_data = new Array();
  236. for(let field in option.data) {
  237. fields_data.push(field);
  238. fields_data.push(option.data[field]);
  239. // 判断是否添加索引
  240. if(option.index.hasOwnProperty(field) == false) continue;
  241. // 添加 set映射
  242. let key_set = this.getKeySet(table, field, option.data[field]);
  243. let expire_set = yield this.getBestExpire(key_set, option.expire);
  244. co(managerMap.redis.sadd(key_set, key_hash_inc, expire_set));
  245. // 判断是否number类型 添加number类型数据 sort set映射
  246. if(option.index[field] == that.type.n) {
  247. let ket_sort_set = this.getKeySortSet(table, field);
  248. let expire_sort_set = yield this.getBestExpire(ket_sort_set, option.expire);
  249. co(managerMap.redis.zadd(ket_sort_set, key_hash_inc, option.data[field], expire_sort_set));
  250. }
  251. }
  252. let expire_hash = yield this.getBestExpire(key_hash, option.expire);
  253. yield managerMap.redis.hmset(key_hash, fields_data, expire_hash);
  254. return {'auto_id':key_hash_inc};
  255. };
  256. /**
  257. * 查询
  258. * @param table
  259. * @param option
  260. * option包括:
  261. * fields 字符串 例如:"name,age" 不能空
  262. * where 字符串 例如"name = ? and age > ?" 不能空 只支持 = > < >= <=
  263. * values 数组 例如['fdl', 2]
  264. * order 字符串 为空默认asc 排序字段必须在fields出现
  265. * limit exsamp:1,2 当获取一条数据时赋值1,为1时返回对象
  266. * @returns {*}
  267. */
  268. this.queryRedisDB = function* (table, option) {
  269. if (F.isNull(table)) F.throwErr('params err: table is null');
  270. if (F.isNull(option)) F.throwErr('params err: option is null');
  271. if (F.isNull(option.fields)) F.throwErr('params err: option.fields is null');
  272. if (F.isNull(option.where)) F.throwErr('params err: option.where is null');
  273. let query_res = yield this.queryResultFromRedisDB(table, option, true);
  274. if(F.isNull(query_res)) {
  275. return [];
  276. }
  277. let fields = option.fields.split(',');
  278. for (let i = 0; i < fields.length; i++) {
  279. fields[i] = _.str.trim(fields[i], ' ');
  280. }
  281. let list = new Array();
  282. let redisCo = managerMap.redis.getRedisCo();
  283. let batchExe = redisCo.multi();
  284. for (let i = 0; i < query_res.length; i++) {
  285. let key_hash = this.getKeyHash(table, query_res[i]);
  286. let param = [key_hash];
  287. param.push.apply(param, fields);
  288. batchExe = batchExe.hmget(param);
  289. }
  290. let res_list = yield batchExe.exec();
  291. for (let i = 0; i < res_list.length; i++) {
  292. let data = res_list[i];
  293. let item = {};
  294. let isAllAtrNull = true;
  295. for (let j = 0; j < fields.length; j++) {
  296. item[fields[j]] = data[j];
  297. if (data[j] != null) isAllAtrNull = false;
  298. }
  299. if (!isAllAtrNull) list.push(item);
  300. }
  301. return list;
  302. };
  303. /**
  304. * 更新
  305. * @param table
  306. * @param option
  307. * option包括:
  308. * where 字符串 例如"name = ? and age > ?" 不能空 只支持 = > < >= <=
  309. * values 数组 例如['fdl', 2]
  310. * data 数据对象
  311. * expire 过期时间 -1为永不过期
  312. * index 数据描述对象 - 设置该值表示添加索引
  313. * field:type type取值: n:number, s:string
  314. * 例如 {name:s, age:n}
  315. * @returns {int} 影响行数
  316. */
  317. this.updateRedisDB = function* (table, option) {
  318. if(F.isNull(table) || F.isNull(option) || F.isNull(option.where) || F.isNull(option.data) || F.isNull(option.expire)
  319. || F.isNull(option.index)) {
  320. F.throwErr('params err.');
  321. }
  322. let query_res = yield this.queryResultFromRedisDB(table, option);
  323. if(F.isNull(query_res)) return 0;
  324. let del_key_array = new Array();
  325. let del_key_record = new Object();
  326. let update_data = new Array();
  327. for (let i = 0; i < query_res.length; i++) {
  328. let key_hash = this.getKeyHash(table, query_res[i]);
  329. let fields_keys = yield managerMap.redis.hkeys(key_hash);
  330. if(F.isNull(fields_keys)) continue;
  331. let fields_data = yield managerMap.redis.hmget(key_hash, fields_keys);
  332. if(F.isNull(fields_data)) continue;
  333. // 查询旧数据字段键值映射 添加到删除数据
  334. let insert_item = new Object();
  335. del_key_array.push(key_hash);
  336. for (let j = 0; j < fields_keys.length; j++) {
  337. if(option.data.hasOwnProperty(fields_keys[j])) { // 只删除更新字段的映射key
  338. // 查询删除key
  339. let key_set = this.getKeySet(table, fields_keys[j], fields_data[j]);
  340. if(F.isNull(del_key_record[key_set])) {
  341. del_key_array.push(key_set);
  342. del_key_record[key_set] = 1;
  343. }
  344. if(typeof(option.data[fields_keys[j]]) == 'number') {
  345. // 删除区间查询映射key
  346. let key_sort_set = this.getKeySortSet(table, fields_keys[j]);
  347. co(managerMap.redis.zrem(this.getKeySortSet(table, fields_keys[j]), query_res[i]));
  348. }
  349. insert_item[fields_keys[j]] = option.data[fields_keys[j]];
  350. }else {
  351. insert_item[fields_keys[j]] = fields_data[j];
  352. }
  353. }
  354. // 检测更新字段中是否存在新增字段
  355. for (let field in option.data) {
  356. if(insert_item.hasOwnProperty(field) == false) {
  357. insert_item[field] = option.data[field];
  358. }
  359. }
  360. update_data.push(insert_item);
  361. }
  362. yield managerMap.redis.del(del_key_array);
  363. // 添加更新的数据
  364. for (let i = 0; i < update_data.length; i++) {
  365. yield this.insertRedisDB(table, {
  366. 'data': update_data[i],
  367. 'expire': option.expire,
  368. 'index':option.index
  369. });
  370. }
  371. return query_res.length;
  372. }
  373. /**
  374. * 删除
  375. * @param table
  376. * @param option
  377. * option包括:
  378. * where 字符串 例如"name = ? and age > ?" 不能空 只支持 = > < >= <=
  379. * @returns {int} 影响行数
  380. */
  381. this.deleteRedisDB = function* (table, option) {
  382. if(F.isNull(table) || F.isNull(option) || F.isNull(option.where)) F.throwErr('params err.');
  383. let query_res = yield this.queryResultFromRedisDB(table, option);
  384. if(F.isNull(query_res)) return 0;
  385. //F.addDebugLogs(["to del:",query_res]);
  386. let del_key_array = new Array();
  387. let del_key_record = new Object();
  388. for (let i = 0; i < query_res.length; i++) {
  389. let key_hash = this.getKeyHash(table, query_res[i]);
  390. let fields_keys = yield managerMap.redis.hkeys(key_hash);
  391. if(F.isNull(fields_keys)) continue;
  392. let fields_data = yield managerMap.redis.hmget(key_hash, fields_keys);
  393. if(F.isNull(fields_data)) continue;
  394. // 查询数据字段键值映射 添加到删除数据
  395. del_key_array.push(key_hash);
  396. //F.addDebugLogs(["to del fields_data:",fields_data]);
  397. for (let j = 0; j < fields_keys.length; j++) {
  398. // 查询删除key
  399. let key_set = this.getKeySet(table, fields_keys[j], fields_data[j]);
  400. //F.addDebugLogs(["to del key_set:",key_set]);
  401. if(F.isNull(del_key_record[key_set])) {
  402. del_key_array.push(key_set);
  403. del_key_record[key_set] = 1;
  404. }
  405. if(this.checkIsNumber(fields_data[j]) == true) {
  406. // 删除区间查询映射key
  407. let key_sort_set = this.getKeySortSet(table, fields_keys[j]);
  408. //F.addDebugLogs(["to del key_sort_set:",key_sort_set,query_res[i],del_key_record[key_sort_set]]);
  409. co(managerMap.redis.zrem(this.getKeySortSet(table, fields_keys[j]), query_res[i]));
  410. }
  411. }
  412. }
  413. yield managerMap.redis.del(del_key_array);
  414. return query_res.length;
  415. }
  416. };