grid_store.js 53 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604
  1. "use strict";
  2. /**
  3. * @fileOverview GridFS is a tool for MongoDB to store files to the database.
  4. * Because of the restrictions of the object size the database can hold, a
  5. * facility to split a file into several chunks is needed. The {@link GridStore}
  6. * class offers a simplified api to interact with files while managing the
  7. * chunks of split files behind the scenes. More information about GridFS can be
  8. * found <a href="http://www.mongodb.org/display/DOCS/GridFS">here</a>.
  9. *
  10. * @example
  11. * var MongoClient = require('mongodb').MongoClient,
  12. * GridStore = require('mongodb').GridStore,
  13. * ObjectID = require('mongodb').ObjectID,
  14. * test = require('assert');
  15. *
  16. * // Connection url
  17. * var url = 'mongodb://localhost:27017/test';
  18. * // Connect using MongoClient
  19. * MongoClient.connect(url, function(err, db) {
  20. * var gridStore = new GridStore(db, null, "w");
  21. * gridStore.open(function(err, gridStore) {
  22. * gridStore.write("hello world!", function(err, gridStore) {
  23. * gridStore.close(function(err, result) {
  24. *
  25. * // Let's read the file using object Id
  26. * GridStore.read(db, result._id, function(err, data) {
  27. * test.equal('hello world!', data);
  28. * db.close();
  29. * test.done();
  30. * });
  31. * });
  32. * });
  33. * });
  34. * });
  35. */
  36. var Chunk = require('./chunk'),
  37. ObjectID = require('mongodb-core').BSON.ObjectID,
  38. ReadPreference = require('../read_preference'),
  39. Buffer = require('buffer').Buffer,
  40. fs = require('fs'),
  41. timers = require('timers'),
  42. f = require('util').format,
  43. util = require('util'),
  44. MongoError = require('mongodb-core').MongoError,
  45. inherits = util.inherits,
  46. Duplex = require('stream').Duplex || require('readable-stream').Duplex;
  47. var REFERENCE_BY_FILENAME = 0,
  48. REFERENCE_BY_ID = 1;
  49. /**
  50. * Namespace provided by the mongodb-core and node.js
  51. * @external Duplex
  52. */
  53. /**
  54. * Create a new GridStore instance
  55. *
  56. * Modes
  57. * - **"r"** - read only. This is the default mode.
  58. * - **"w"** - write in truncate mode. Existing data will be overwriten.
  59. *
  60. * @class
  61. * @param {Db} db A database instance to interact with.
  62. * @param {object} [id] optional unique id for this file
  63. * @param {string} [filename] optional filename for this file, no unique constrain on the field
  64. * @param {string} mode set the mode for this file.
  65. * @param {object} [options=null] Optional settings.
  66. * @param {(number|string)} [options.w=null] The write concern.
  67. * @param {number} [options.wtimeout=null] The write concern timeout.
  68. * @param {boolean} [options.j=false] Specify a journal write concern.
  69. * @param {boolean} [options.fsync=false] Specify a file sync write concern.
  70. * @param {string} [options.root=null] Root collection to use. Defaults to **{GridStore.DEFAULT_ROOT_COLLECTION}**.
  71. * @param {string} [options.content_type=null] MIME type of the file. Defaults to **{GridStore.DEFAULT_CONTENT_TYPE}**.
  72. * @param {number} [options.chunk_size=261120] Size for the chunk. Defaults to **{Chunk.DEFAULT_CHUNK_SIZE}**.
  73. * @param {object} [options.metadata=null] Arbitrary data the user wants to store.
  74. * @param {(ReadPreference|string)} [options.readPreference=null] The preferred read preference (ReadPreference.PRIMARY, ReadPreference.PRIMARY_PREFERRED, ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED, ReadPreference.NEAREST).
  75. * @property {number} chunkSize Get the gridstore chunk size.
  76. * @property {number} md5 The md5 checksum for this file.
  77. * @property {number} chunkNumber The current chunk number the gridstore has materialized into memory
  78. * @return {GridStore} a GridStore instance.
  79. */
  80. var GridStore = function GridStore(db, id, filename, mode, options) {
  81. if(!(this instanceof GridStore)) return new GridStore(db, id, filename, mode, options);
  82. var self = this;
  83. this.db = db;
  84. // Handle options
  85. if(typeof options === 'undefined') options = {};
  86. // Handle mode
  87. if(typeof mode === 'undefined') {
  88. mode = filename;
  89. filename = undefined;
  90. } else if(typeof mode == 'object') {
  91. options = mode;
  92. mode = filename;
  93. filename = undefined;
  94. }
  95. if(id instanceof ObjectID) {
  96. this.referenceBy = REFERENCE_BY_ID;
  97. this.fileId = id;
  98. this.filename = filename;
  99. } else if(typeof filename == 'undefined') {
  100. this.referenceBy = REFERENCE_BY_FILENAME;
  101. this.filename = id;
  102. if (mode.indexOf('w') != null) {
  103. this.fileId = new ObjectID();
  104. }
  105. } else {
  106. this.referenceBy = REFERENCE_BY_ID;
  107. this.fileId = id;
  108. this.filename = filename;
  109. }
  110. // Set up the rest
  111. this.mode = mode == null ? "r" : mode;
  112. this.options = options || {};
  113. // Opened
  114. this.isOpen = false;
  115. // Set the root if overridden
  116. this.root = this.options['root'] == null ? GridStore.DEFAULT_ROOT_COLLECTION : this.options['root'];
  117. this.position = 0;
  118. this.readPreference = this.options.readPreference || ReadPreference.PRIMARY;
  119. this.writeConcern = _getWriteConcern(db, this.options);
  120. // Set default chunk size
  121. this.internalChunkSize = this.options['chunkSize'] == null ? Chunk.DEFAULT_CHUNK_SIZE : this.options['chunkSize'];
  122. Object.defineProperty(this, "chunkSize", { enumerable: true
  123. , get: function () {
  124. return this.internalChunkSize;
  125. }
  126. , set: function(value) {
  127. if(!(this.mode[0] == "w" && this.position == 0 && this.uploadDate == null)) {
  128. this.internalChunkSize = this.internalChunkSize;
  129. } else {
  130. this.internalChunkSize = value;
  131. }
  132. }
  133. });
  134. Object.defineProperty(this, "md5", { enumerable: true
  135. , get: function () {
  136. return this.internalMd5;
  137. }
  138. });
  139. Object.defineProperty(this, "chunkNumber", { enumerable: true
  140. , get: function () {
  141. return this.currentChunk && this.currentChunk.chunkNumber ? this.currentChunk.chunkNumber : null;
  142. }
  143. });
  144. }
  145. /**
  146. * The callback format for the Gridstore.open method
  147. * @callback GridStore~openCallback
  148. * @param {MongoError} error An error instance representing the error during the execution.
  149. * @param {GridStore} gridStore The GridStore instance if the open method was successful.
  150. */
  151. /**
  152. * Opens the file from the database and initialize this object. Also creates a
  153. * new one if file does not exist.
  154. *
  155. * @method
  156. * @param {GridStore~openCallback} callback this will be called after executing this method
  157. * @return {null}
  158. */
  159. GridStore.prototype.open = function(callback) {
  160. if( this.mode != "w" && this.mode != "w+" && this.mode != "r"){
  161. callback(new MongoError("Illegal mode " + this.mode), null);
  162. return;
  163. }
  164. var self = this;
  165. // Get the write concern
  166. var writeConcern = _getWriteConcern(this.db, this.options);
  167. // If we are writing we need to ensure we have the right indexes for md5's
  168. if((self.mode == "w" || self.mode == "w+")) {
  169. // Get files collection
  170. var collection = self.collection();
  171. // Put index on filename
  172. collection.ensureIndex([['filename', 1]], writeConcern, function(err, index) {
  173. // Get chunk collection
  174. var chunkCollection = self.chunkCollection();
  175. // Ensure index on chunk collection
  176. chunkCollection.ensureIndex([['files_id', 1], ['n', 1]], writeConcern, function(err, index) {
  177. // Open the connection
  178. _open(self, writeConcern, function(err, r) {
  179. if(err) return callback(err);
  180. self.isOpen = true;
  181. callback(err, r);
  182. });
  183. });
  184. });
  185. } else {
  186. // Open the gridstore
  187. _open(self, writeConcern, function(err, r) {
  188. if(err) return callback(err);
  189. self.isOpen = true;
  190. callback(err, r);
  191. });
  192. }
  193. };
  194. /**
  195. * Verify if the file is at EOF.
  196. *
  197. * @method
  198. * @return {boolean} true if the read/write head is at the end of this file.
  199. */
  200. GridStore.prototype.eof = function() {
  201. return this.position == this.length ? true : false;
  202. }
  203. /**
  204. * The callback result format.
  205. * @callback GridStore~resultCallback
  206. * @param {MongoError} error An error instance representing the error during the execution.
  207. * @param {object} result The result from the callback.
  208. */
  209. /**
  210. * Retrieves a single character from this file.
  211. *
  212. * @method
  213. * @param {GridStore~resultCallback} callback this gets called after this method is executed. Passes null to the first parameter and the character read to the second or null to the second if the read/write head is at the end of the file.
  214. * @return {null}
  215. */
  216. GridStore.prototype.getc = function(callback) {
  217. var self = this;
  218. if(self.eof()) {
  219. callback(null, null);
  220. } else if(self.currentChunk.eof()) {
  221. nthChunk(self, self.currentChunk.chunkNumber + 1, function(err, chunk) {
  222. self.currentChunk = chunk;
  223. self.position = self.position + 1;
  224. callback(err, self.currentChunk.getc());
  225. });
  226. } else {
  227. self.position = self.position + 1;
  228. callback(null, self.currentChunk.getc());
  229. }
  230. }
  231. /**
  232. * Writes a string to the file with a newline character appended at the end if
  233. * the given string does not have one.
  234. *
  235. * @method
  236. * @param {string} string the string to write.
  237. * @param {GridStore~resultCallback} callback this will be called after executing this method. The first parameter will contain null and the second one will contain a reference to this object.
  238. * @return {null}
  239. */
  240. GridStore.prototype.puts = function(string, callback) {
  241. var finalString = string.match(/\n$/) == null ? string + "\n" : string;
  242. this.write(finalString, callback);
  243. }
  244. /**
  245. * Return a modified Readable stream including a possible transform method.
  246. *
  247. * @method
  248. * @return {GridStoreStream}
  249. */
  250. GridStore.prototype.stream = function() {
  251. return new GridStoreStream(this);
  252. }
  253. /**
  254. * Writes some data. This method will work properly only if initialized with mode "w" or "w+".
  255. *
  256. * @method
  257. * @param {(string|Buffer)} data the data to write.
  258. * @param {boolean} [close] closes this file after writing if set to true.
  259. * @param {GridStore~resultCallback} callback this will be called after executing this method. The first parameter will contain null and the second one will contain a reference to this object.
  260. * @return {null}
  261. */
  262. GridStore.prototype.write = function write(data, close, callback) {
  263. return _writeNormal(this, data, close, callback);
  264. }
  265. /**
  266. * Handles the destroy part of a stream
  267. *
  268. * @method
  269. * @result {null}
  270. */
  271. GridStore.prototype.destroy = function destroy() {
  272. // close and do not emit any more events. queued data is not sent.
  273. if(!this.writable) return;
  274. this.readable = false;
  275. if(this.writable) {
  276. this.writable = false;
  277. this._q.length = 0;
  278. this.emit('close');
  279. }
  280. }
  281. /**
  282. * Stores a file from the file system to the GridFS database.
  283. *
  284. * @method
  285. * @param {(string|Buffer|FileHandle)} file the file to store.
  286. * @param {GridStore~resultCallback} callback this will be called after executing this method. The first parameter will contain null and the second one will contain a reference to this object.
  287. * @return {null}
  288. */
  289. GridStore.prototype.writeFile = function (file, callback) {
  290. var self = this;
  291. if (typeof file === 'string') {
  292. fs.open(file, 'r', function (err, fd) {
  293. if(err) return callback(err);
  294. self.writeFile(fd, callback);
  295. });
  296. return;
  297. }
  298. self.open(function (err, self) {
  299. if(err) return callback(err, self);
  300. fs.fstat(file, function (err, stats) {
  301. if(err) return callback(err, self);
  302. var offset = 0;
  303. var index = 0;
  304. var numberOfChunksLeft = Math.min(stats.size / self.chunkSize);
  305. // Write a chunk
  306. var writeChunk = function() {
  307. fs.read(file, self.chunkSize, offset, 'binary', function(err, data, bytesRead) {
  308. if(err) return callback(err, self);
  309. offset = offset + bytesRead;
  310. // Create a new chunk for the data
  311. var chunk = new Chunk(self, {n:index++}, self.writeConcern);
  312. chunk.write(data, function(err, chunk) {
  313. if(err) return callback(err, self);
  314. chunk.save({}, function(err, result) {
  315. if(err) return callback(err, self);
  316. self.position = self.position + data.length;
  317. // Point to current chunk
  318. self.currentChunk = chunk;
  319. if(offset >= stats.size) {
  320. fs.close(file);
  321. self.close(function(err, result) {
  322. if(err) return callback(err, self);
  323. return callback(null, self);
  324. });
  325. } else {
  326. return process.nextTick(writeChunk);
  327. }
  328. });
  329. });
  330. });
  331. }
  332. // Process the first write
  333. process.nextTick(writeChunk);
  334. });
  335. });
  336. };
  337. /**
  338. * Saves this file to the database. This will overwrite the old entry if it
  339. * already exists. This will work properly only if mode was initialized to
  340. * "w" or "w+".
  341. *
  342. * @method
  343. * @param {GridStore~resultCallback} callback this will be called after executing this method. The first parameter will contain null and the second one will contain a reference to this object.
  344. * @return {null}
  345. */
  346. GridStore.prototype.close = function(callback) {
  347. var self = this;
  348. if(self.mode[0] == "w") {
  349. // Set up options
  350. var options = self.writeConcern;
  351. if(self.currentChunk != null && self.currentChunk.position > 0) {
  352. self.currentChunk.save({}, function(err, chunk) {
  353. if(err && typeof callback == 'function') return callback(err);
  354. self.collection(function(err, files) {
  355. if(err && typeof callback == 'function') return callback(err);
  356. // Build the mongo object
  357. if(self.uploadDate != null) {
  358. files.remove({'_id':self.fileId}, self.writeConcern, function(err, collection) {
  359. if(err && typeof callback == 'function') return callback(err);
  360. buildMongoObject(self, function(err, mongoObject) {
  361. if(err) {
  362. if(typeof callback == 'function') return callback(err); else throw err;
  363. }
  364. files.save(mongoObject, options, function(err) {
  365. if(typeof callback == 'function')
  366. callback(err, mongoObject);
  367. });
  368. });
  369. });
  370. } else {
  371. self.uploadDate = new Date();
  372. buildMongoObject(self, function(err, mongoObject) {
  373. if(err) {
  374. if(typeof callback == 'function') return callback(err); else throw err;
  375. }
  376. files.save(mongoObject, options, function(err) {
  377. if(typeof callback == 'function')
  378. callback(err, mongoObject);
  379. });
  380. });
  381. }
  382. });
  383. });
  384. } else {
  385. self.collection(function(err, files) {
  386. if(err && typeof callback == 'function') return callback(err);
  387. self.uploadDate = new Date();
  388. buildMongoObject(self, function(err, mongoObject) {
  389. if(err) {
  390. if(typeof callback == 'function') return callback(err); else throw err;
  391. }
  392. files.save(mongoObject, options, function(err) {
  393. if(typeof callback == 'function')
  394. callback(err, mongoObject);
  395. });
  396. });
  397. });
  398. }
  399. } else if(self.mode[0] == "r") {
  400. if(typeof callback == 'function')
  401. callback(null, null);
  402. } else {
  403. if(typeof callback == 'function')
  404. callback(new MongoError(f("Illegal mode %s", self.mode), null));
  405. }
  406. };
  407. /**
  408. * The collection callback format.
  409. * @callback GridStore~collectionCallback
  410. * @param {MongoError} error An error instance representing the error during the execution.
  411. * @param {Collection} collection The collection from the command execution.
  412. */
  413. /**
  414. * Retrieve this file's chunks collection.
  415. *
  416. * @method
  417. * @param {GridStore~collectionCallback} callback the command callback.
  418. * @return {Collection}
  419. */
  420. GridStore.prototype.chunkCollection = function(callback) {
  421. if(typeof callback == 'function')
  422. return this.db.collection((this.root + ".chunks"), callback);
  423. return this.db.collection((this.root + ".chunks"));
  424. };
  425. /**
  426. * Deletes all the chunks of this file in the database.
  427. *
  428. * @method
  429. * @param {GridStore~resultCallback} callback the command callback.
  430. * @return {null}
  431. */
  432. GridStore.prototype.unlink = function(callback) {
  433. var self = this;
  434. deleteChunks(this, function(err) {
  435. if(err!==null) {
  436. err.message = "at deleteChunks: " + err.message;
  437. return callback(err);
  438. }
  439. self.collection(function(err, collection) {
  440. if(err!==null) {
  441. err.message = "at collection: " + err.message;
  442. return callback(err);
  443. }
  444. collection.remove({'_id':self.fileId}, self.writeConcern, function(err) {
  445. callback(err, self);
  446. });
  447. });
  448. });
  449. };
  450. /**
  451. * Retrieves the file collection associated with this object.
  452. *
  453. * @method
  454. * @param {GridStore~collectionCallback} callback the command callback.
  455. * @return {Collection}
  456. */
  457. GridStore.prototype.collection = function(callback) {
  458. if(typeof callback == 'function')
  459. this.db.collection(this.root + ".files", callback);
  460. return this.db.collection(this.root + ".files");
  461. };
  462. /**
  463. * The readlines callback format.
  464. * @callback GridStore~readlinesCallback
  465. * @param {MongoError} error An error instance representing the error during the execution.
  466. * @param {string[]} strings The array of strings returned.
  467. */
  468. /**
  469. * Read the entire file as a list of strings splitting by the provided separator.
  470. *
  471. * @method
  472. * @param {string} [separator] The character to be recognized as the newline separator.
  473. * @param {GridStore~readlinesCallback} callback the command callback.
  474. * @return {null}
  475. */
  476. GridStore.prototype.readlines = function(separator, callback) {
  477. var args = Array.prototype.slice.call(arguments, 0);
  478. callback = args.pop();
  479. separator = args.length ? args.shift() : "\n";
  480. this.read(function(err, data) {
  481. if(err) return callback(err);
  482. var items = data.toString().split(separator);
  483. items = items.length > 0 ? items.splice(0, items.length - 1) : [];
  484. for(var i = 0; i < items.length; i++) {
  485. items[i] = items[i] + separator;
  486. }
  487. callback(null, items);
  488. });
  489. };
  490. /**
  491. * Deletes all the chunks of this file in the database if mode was set to "w" or
  492. * "w+" and resets the read/write head to the initial position.
  493. *
  494. * @method
  495. * @param {GridStore~resultCallback} callback this will be called after executing this method. The first parameter will contain null and the second one will contain a reference to this object.
  496. * @return {null}
  497. */
  498. GridStore.prototype.rewind = function(callback) {
  499. var self = this;
  500. if(this.currentChunk.chunkNumber != 0) {
  501. if(this.mode[0] == "w") {
  502. deleteChunks(self, function(err, gridStore) {
  503. if(err) return callback(err);
  504. self.currentChunk = new Chunk(self, {'n': 0}, self.writeConcern);
  505. self.position = 0;
  506. callback(null, self);
  507. });
  508. } else {
  509. self.currentChunk(0, function(err, chunk) {
  510. if(err) return callback(err);
  511. self.currentChunk = chunk;
  512. self.currentChunk.rewind();
  513. self.position = 0;
  514. callback(null, self);
  515. });
  516. }
  517. } else {
  518. self.currentChunk.rewind();
  519. self.position = 0;
  520. callback(null, self);
  521. }
  522. };
  523. /**
  524. * The read callback format.
  525. * @callback GridStore~readCallback
  526. * @param {MongoError} error An error instance representing the error during the execution.
  527. * @param {Buffer} data The data read from the GridStore object
  528. */
  529. /**
  530. * Retrieves the contents of this file and advances the read/write head. Works with Buffers only.
  531. *
  532. * There are 3 signatures for this method:
  533. *
  534. * (callback)
  535. * (length, callback)
  536. * (length, buffer, callback)
  537. *
  538. * @method
  539. * @param {number} [length] the number of characters to read. Reads all the characters from the read/write head to the EOF if not specified.
  540. * @param {(string|Buffer)} [buffer] a string to hold temporary data. This is used for storing the string data read so far when recursively calling this method.
  541. * @param {GridStore~readCallback} callback the command callback.
  542. * @return {null}
  543. */
  544. GridStore.prototype.read = function(length, buffer, callback) {
  545. var self = this;
  546. var args = Array.prototype.slice.call(arguments, 0);
  547. callback = args.pop();
  548. length = args.length ? args.shift() : null;
  549. buffer = args.length ? args.shift() : null;
  550. // The data is a c-terminated string and thus the length - 1
  551. var finalLength = length == null ? self.length - self.position : length;
  552. var finalBuffer = buffer == null ? new Buffer(finalLength) : buffer;
  553. // Add a index to buffer to keep track of writing position or apply current index
  554. finalBuffer._index = buffer != null && buffer._index != null ? buffer._index : 0;
  555. if((self.currentChunk.length() - self.currentChunk.position + finalBuffer._index) >= finalLength) {
  556. var slice = self.currentChunk.readSlice(finalLength - finalBuffer._index);
  557. // Copy content to final buffer
  558. slice.copy(finalBuffer, finalBuffer._index);
  559. // Update internal position
  560. self.position = self.position + finalBuffer.length;
  561. // Check if we don't have a file at all
  562. if(finalLength == 0 && finalBuffer.length == 0) return callback(new MongoError("File does not exist"), null);
  563. // Else return data
  564. return callback(null, finalBuffer);
  565. }
  566. // Read the next chunk
  567. var slice = self.currentChunk.readSlice(self.currentChunk.length() - self.currentChunk.position);
  568. // Copy content to final buffer
  569. slice.copy(finalBuffer, finalBuffer._index);
  570. // Update index position
  571. finalBuffer._index += slice.length;
  572. // Load next chunk and read more
  573. nthChunk(self, self.currentChunk.chunkNumber + 1, function(err, chunk) {
  574. if(err) return callback(err);
  575. if(chunk.length() > 0) {
  576. self.currentChunk = chunk;
  577. self.read(length, finalBuffer, callback);
  578. } else {
  579. if(finalBuffer._index > 0) {
  580. callback(null, finalBuffer)
  581. } else {
  582. callback(new MongoError("no chunks found for file, possibly corrupt"), null);
  583. }
  584. }
  585. });
  586. }
  587. /**
  588. * The tell callback format.
  589. * @callback GridStore~tellCallback
  590. * @param {MongoError} error An error instance representing the error during the execution.
  591. * @param {number} position The current read position in the GridStore.
  592. */
  593. /**
  594. * Retrieves the position of the read/write head of this file.
  595. *
  596. * @method
  597. * @param {number} [length] the number of characters to read. Reads all the characters from the read/write head to the EOF if not specified.
  598. * @param {(string|Buffer)} [buffer] a string to hold temporary data. This is used for storing the string data read so far when recursively calling this method.
  599. * @param {GridStore~tellCallback} callback the command callback.
  600. * @return {null}
  601. */
  602. GridStore.prototype.tell = function(callback) {
  603. callback(null, this.position);
  604. };
  605. /**
  606. * The tell callback format.
  607. * @callback GridStore~gridStoreCallback
  608. * @param {MongoError} error An error instance representing the error during the execution.
  609. * @param {GridStore} gridStore The gridStore.
  610. */
  611. /**
  612. * Moves the read/write head to a new location.
  613. *
  614. * There are 3 signatures for this method
  615. *
  616. * Seek Location Modes
  617. * - **GridStore.IO_SEEK_SET**, **(default)** set the position from the start of the file.
  618. * - **GridStore.IO_SEEK_CUR**, set the position from the current position in the file.
  619. * - **GridStore.IO_SEEK_END**, set the position from the end of the file.
  620. *
  621. * @method
  622. * @param {number} [position] the position to seek to
  623. * @param {number} [seekLocation] seek mode. Use one of the Seek Location modes.
  624. * @param {GridStore~gridStoreCallback} callback the command callback.
  625. * @return {null}
  626. */
  627. GridStore.prototype.seek = function(position, seekLocation, callback) {
  628. var self = this;
  629. var args = Array.prototype.slice.call(arguments, 1);
  630. callback = args.pop();
  631. seekLocation = args.length ? args.shift() : null;
  632. // Seek only supports read mode
  633. if(self.mode != 'r') {
  634. return callback(new MongoError("seek is only supported for mode r"))
  635. }
  636. var seekLocationFinal = seekLocation == null ? GridStore.IO_SEEK_SET : seekLocation;
  637. var finalPosition = position;
  638. var targetPosition = 0;
  639. // Calculate the position
  640. if(seekLocationFinal == GridStore.IO_SEEK_CUR) {
  641. targetPosition = self.position + finalPosition;
  642. } else if(seekLocationFinal == GridStore.IO_SEEK_END) {
  643. targetPosition = self.length + finalPosition;
  644. } else {
  645. targetPosition = finalPosition;
  646. }
  647. // Get the chunk
  648. var newChunkNumber = Math.floor(targetPosition/self.chunkSize);
  649. var seekChunk = function() {
  650. nthChunk(self, newChunkNumber, function(err, chunk) {
  651. self.currentChunk = chunk;
  652. self.position = targetPosition;
  653. self.currentChunk.position = (self.position % self.chunkSize);
  654. callback(err, self);
  655. });
  656. };
  657. seekChunk();
  658. }
  659. /**
  660. * @ignore
  661. */
  662. var _open = function(self, options, callback) {
  663. var collection = self.collection();
  664. // Create the query
  665. var query = self.referenceBy == REFERENCE_BY_ID ? {_id:self.fileId} : {filename:self.filename};
  666. query = null == self.fileId && self.filename == null ? null : query;
  667. options.readPreference = self.readPreference;
  668. // Fetch the chunks
  669. if(query != null) {
  670. collection.findOne(query, options, function(err, doc) {
  671. if(err) return error(err);
  672. // Check if the collection for the files exists otherwise prepare the new one
  673. if(doc != null) {
  674. self.fileId = doc._id;
  675. // Prefer a new filename over the existing one if this is a write
  676. self.filename = ((self.mode == 'r') || (self.filename == undefined)) ? doc.filename : self.filename;
  677. self.contentType = doc.contentType;
  678. self.internalChunkSize = doc.chunkSize;
  679. self.uploadDate = doc.uploadDate;
  680. self.aliases = doc.aliases;
  681. self.length = doc.length;
  682. self.metadata = doc.metadata;
  683. self.internalMd5 = doc.md5;
  684. } else if (self.mode != 'r') {
  685. self.fileId = self.fileId == null ? new ObjectID() : self.fileId;
  686. self.contentType = GridStore.DEFAULT_CONTENT_TYPE;
  687. self.internalChunkSize = self.internalChunkSize == null ? Chunk.DEFAULT_CHUNK_SIZE : self.internalChunkSize;
  688. self.length = 0;
  689. } else {
  690. self.length = 0;
  691. var txtId = self.fileId instanceof ObjectID ? self.fileId.toHexString() : self.fileId;
  692. return error(new MongoError(f("file with id %s not opened for writing", (self.referenceBy == REFERENCE_BY_ID ? txtId : self.filename))), self);
  693. }
  694. // Process the mode of the object
  695. if(self.mode == "r") {
  696. nthChunk(self, 0, options, function(err, chunk) {
  697. if(err) return error(err);
  698. self.currentChunk = chunk;
  699. self.position = 0;
  700. callback(null, self);
  701. });
  702. } else if(self.mode == "w") {
  703. // Delete any existing chunks
  704. deleteChunks(self, options, function(err, result) {
  705. if(err) return error(err);
  706. self.currentChunk = new Chunk(self, {'n':0}, self.writeConcern);
  707. self.contentType = self.options['content_type'] == null ? self.contentType : self.options['content_type'];
  708. self.internalChunkSize = self.options['chunk_size'] == null ? self.internalChunkSize : self.options['chunk_size'];
  709. self.metadata = self.options['metadata'] == null ? self.metadata : self.options['metadata'];
  710. self.aliases = self.options['aliases'] == null ? self.aliases : self.options['aliases'];
  711. self.position = 0;
  712. callback(null, self);
  713. });
  714. } else if(self.mode == "w+") {
  715. nthChunk(self, lastChunkNumber(self), options, function(err, chunk) {
  716. if(err) return error(err);
  717. // Set the current chunk
  718. self.currentChunk = chunk == null ? new Chunk(self, {'n':0}, self.writeConcern) : chunk;
  719. self.currentChunk.position = self.currentChunk.data.length();
  720. self.metadata = self.options['metadata'] == null ? self.metadata : self.options['metadata'];
  721. self.aliases = self.options['aliases'] == null ? self.aliases : self.options['aliases'];
  722. self.position = self.length;
  723. callback(null, self);
  724. });
  725. }
  726. });
  727. } else {
  728. // Write only mode
  729. self.fileId = null == self.fileId ? new ObjectID() : self.fileId;
  730. self.contentType = GridStore.DEFAULT_CONTENT_TYPE;
  731. self.internalChunkSize = self.internalChunkSize == null ? Chunk.DEFAULT_CHUNK_SIZE : self.internalChunkSize;
  732. self.length = 0;
  733. var collection2 = self.chunkCollection();
  734. // No file exists set up write mode
  735. if(self.mode == "w") {
  736. // Delete any existing chunks
  737. deleteChunks(self, options, function(err, result) {
  738. if(err) return error(err);
  739. self.currentChunk = new Chunk(self, {'n':0}, self.writeConcern);
  740. self.contentType = self.options['content_type'] == null ? self.contentType : self.options['content_type'];
  741. self.internalChunkSize = self.options['chunk_size'] == null ? self.internalChunkSize : self.options['chunk_size'];
  742. self.metadata = self.options['metadata'] == null ? self.metadata : self.options['metadata'];
  743. self.aliases = self.options['aliases'] == null ? self.aliases : self.options['aliases'];
  744. self.position = 0;
  745. callback(null, self);
  746. });
  747. } else if(self.mode == "w+") {
  748. nthChunk(self, lastChunkNumber(self), options, function(err, chunk) {
  749. if(err) return error(err);
  750. // Set the current chunk
  751. self.currentChunk = chunk == null ? new Chunk(self, {'n':0}, self.writeConcern) : chunk;
  752. self.currentChunk.position = self.currentChunk.data.length();
  753. self.metadata = self.options['metadata'] == null ? self.metadata : self.options['metadata'];
  754. self.aliases = self.options['aliases'] == null ? self.aliases : self.options['aliases'];
  755. self.position = self.length;
  756. callback(null, self);
  757. });
  758. }
  759. }
  760. // only pass error to callback once
  761. function error (err) {
  762. if(error.err) return;
  763. callback(error.err = err);
  764. }
  765. };
  766. /**
  767. * @ignore
  768. */
  769. var writeBuffer = function(self, buffer, close, callback) {
  770. if(typeof close === "function") { callback = close; close = null; }
  771. var finalClose = typeof close == 'boolean' ? close : false;
  772. if(self.mode != "w") {
  773. callback(new MongoError(f("file with id %s not opened for writing", (self.referenceBy == REFERENCE_BY_ID ? self.referenceBy : self.filename))), null);
  774. } else {
  775. if(self.currentChunk.position + buffer.length >= self.chunkSize) {
  776. // Write out the current Chunk and then keep writing until we have less data left than a chunkSize left
  777. // to a new chunk (recursively)
  778. var previousChunkNumber = self.currentChunk.chunkNumber;
  779. var leftOverDataSize = self.chunkSize - self.currentChunk.position;
  780. var firstChunkData = buffer.slice(0, leftOverDataSize);
  781. var leftOverData = buffer.slice(leftOverDataSize);
  782. // A list of chunks to write out
  783. var chunksToWrite = [self.currentChunk.write(firstChunkData)];
  784. // If we have more data left than the chunk size let's keep writing new chunks
  785. while(leftOverData.length >= self.chunkSize) {
  786. // Create a new chunk and write to it
  787. var newChunk = new Chunk(self, {'n': (previousChunkNumber + 1)}, self.writeConcern);
  788. var firstChunkData = leftOverData.slice(0, self.chunkSize);
  789. leftOverData = leftOverData.slice(self.chunkSize);
  790. // Update chunk number
  791. previousChunkNumber = previousChunkNumber + 1;
  792. // Write data
  793. newChunk.write(firstChunkData);
  794. // Push chunk to save list
  795. chunksToWrite.push(newChunk);
  796. }
  797. // Set current chunk with remaining data
  798. self.currentChunk = new Chunk(self, {'n': (previousChunkNumber + 1)}, self.writeConcern);
  799. // If we have left over data write it
  800. if(leftOverData.length > 0) self.currentChunk.write(leftOverData);
  801. // Update the position for the gridstore
  802. self.position = self.position + buffer.length;
  803. // Total number of chunks to write
  804. var numberOfChunksToWrite = chunksToWrite.length;
  805. for(var i = 0; i < chunksToWrite.length; i++) {
  806. chunksToWrite[i].save({}, function(err, result) {
  807. if(err) return callback(err);
  808. numberOfChunksToWrite = numberOfChunksToWrite - 1;
  809. if(numberOfChunksToWrite <= 0) {
  810. // We care closing the file before returning
  811. if(finalClose) {
  812. return self.close(function(err, result) {
  813. callback(err, self);
  814. });
  815. }
  816. // Return normally
  817. return callback(null, self);
  818. }
  819. });
  820. }
  821. } else {
  822. // Update the position for the gridstore
  823. self.position = self.position + buffer.length;
  824. // We have less data than the chunk size just write it and callback
  825. self.currentChunk.write(buffer);
  826. // We care closing the file before returning
  827. if(finalClose) {
  828. return self.close(function(err, result) {
  829. callback(err, self);
  830. });
  831. }
  832. // Return normally
  833. return callback(null, self);
  834. }
  835. }
  836. };
  837. /**
  838. * Creates a mongoDB object representation of this object.
  839. *
  840. * <pre><code>
  841. * {
  842. * '_id' : , // {number} id for this file
  843. * 'filename' : , // {string} name for this file
  844. * 'contentType' : , // {string} mime type for this file
  845. * 'length' : , // {number} size of this file?
  846. * 'chunksize' : , // {number} chunk size used by this file
  847. * 'uploadDate' : , // {Date}
  848. * 'aliases' : , // {array of string}
  849. * 'metadata' : , // {string}
  850. * }
  851. * </code></pre>
  852. *
  853. * @ignore
  854. */
  855. var buildMongoObject = function(self, callback) {
  856. // Calcuate the length
  857. var mongoObject = {
  858. '_id': self.fileId,
  859. 'filename': self.filename,
  860. 'contentType': self.contentType,
  861. 'length': self.position ? self.position : 0,
  862. 'chunkSize': self.chunkSize,
  863. 'uploadDate': self.uploadDate,
  864. 'aliases': self.aliases,
  865. 'metadata': self.metadata
  866. };
  867. var md5Command = {filemd5:self.fileId, root:self.root};
  868. self.db.command(md5Command, function(err, results) {
  869. if(err) return callback(err);
  870. mongoObject.md5 = results.md5;
  871. callback(null, mongoObject);
  872. });
  873. };
  874. /**
  875. * Gets the nth chunk of this file.
  876. * @ignore
  877. */
  878. var nthChunk = function(self, chunkNumber, options, callback) {
  879. if(typeof options == 'function') {
  880. callback = options;
  881. options = {};
  882. }
  883. options = options || self.writeConcern;
  884. options.readPreference = self.readPreference;
  885. // Get the nth chunk
  886. self.chunkCollection().findOne({'files_id':self.fileId, 'n':chunkNumber}, options, function(err, chunk) {
  887. if(err) return callback(err);
  888. var finalChunk = chunk == null ? {} : chunk;
  889. callback(null, new Chunk(self, finalChunk, self.writeConcern));
  890. });
  891. };
  892. /**
  893. * @ignore
  894. */
  895. var lastChunkNumber = function(self) {
  896. return Math.floor((self.length ? self.length - 1 : 0)/self.chunkSize);
  897. };
  898. /**
  899. * Deletes all the chunks of this file in the database.
  900. *
  901. * @ignore
  902. */
  903. var deleteChunks = function(self, options, callback) {
  904. if(typeof options == 'function') {
  905. callback = options;
  906. options = {};
  907. }
  908. options = options || self.writeConcern;
  909. if(self.fileId != null) {
  910. self.chunkCollection().remove({'files_id':self.fileId}, options, function(err, result) {
  911. if(err) return callback(err, false);
  912. callback(null, true);
  913. });
  914. } else {
  915. callback(null, true);
  916. }
  917. };
  918. /**
  919. * The collection to be used for holding the files and chunks collection.
  920. *
  921. * @classconstant DEFAULT_ROOT_COLLECTION
  922. **/
  923. GridStore.DEFAULT_ROOT_COLLECTION = 'fs';
  924. /**
  925. * Default file mime type
  926. *
  927. * @classconstant DEFAULT_CONTENT_TYPE
  928. **/
  929. GridStore.DEFAULT_CONTENT_TYPE = 'binary/octet-stream';
  930. /**
  931. * Seek mode where the given length is absolute.
  932. *
  933. * @classconstant IO_SEEK_SET
  934. **/
  935. GridStore.IO_SEEK_SET = 0;
  936. /**
  937. * Seek mode where the given length is an offset to the current read/write head.
  938. *
  939. * @classconstant IO_SEEK_CUR
  940. **/
  941. GridStore.IO_SEEK_CUR = 1;
  942. /**
  943. * Seek mode where the given length is an offset to the end of the file.
  944. *
  945. * @classconstant IO_SEEK_END
  946. **/
  947. GridStore.IO_SEEK_END = 2;
  948. /**
  949. * Checks if a file exists in the database.
  950. *
  951. * @method
  952. * @static
  953. * @param {Db} db the database to query.
  954. * @param {string} name The name of the file to look for.
  955. * @param {string} [rootCollection] The root collection that holds the files and chunks collection. Defaults to **{GridStore.DEFAULT_ROOT_COLLECTION}**.
  956. * @param {object} [options=null] Optional settings.
  957. * @param {(ReadPreference|string)} [options.readPreference=null] The preferred read preference (ReadPreference.PRIMARY, ReadPreference.PRIMARY_PREFERRED, ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED, ReadPreference.NEAREST).
  958. * @param {GridStore~resultCallback} callback result from exists.
  959. * @return {null}
  960. */
  961. GridStore.exist = function(db, fileIdObject, rootCollection, options, callback) {
  962. var args = Array.prototype.slice.call(arguments, 2);
  963. callback = args.pop();
  964. rootCollection = args.length ? args.shift() : null;
  965. options = args.length ? args.shift() : {};
  966. // Establish read preference
  967. var readPreference = options.readPreference || ReadPreference.PRIMARY;
  968. // Fetch collection
  969. var rootCollectionFinal = rootCollection != null ? rootCollection : GridStore.DEFAULT_ROOT_COLLECTION;
  970. db.collection(rootCollectionFinal + ".files", function(err, collection) {
  971. if(err) return callback(err);
  972. // Build query
  973. var query = (typeof fileIdObject == 'string' || Object.prototype.toString.call(fileIdObject) == '[object RegExp]' )
  974. ? {'filename':fileIdObject}
  975. : {'_id':fileIdObject}; // Attempt to locate file
  976. // We have a specific query
  977. if(fileIdObject != null
  978. && typeof fileIdObject == 'object'
  979. && Object.prototype.toString.call(fileIdObject) != '[object RegExp]') {
  980. query = fileIdObject;
  981. }
  982. // Check if the entry exists
  983. collection.findOne(query, {readPreference:readPreference}, function(err, item) {
  984. if(err) return callback(err);
  985. callback(null, item == null ? false : true);
  986. });
  987. });
  988. };
  989. /**
  990. * Gets the list of files stored in the GridFS.
  991. *
  992. * @method
  993. * @static
  994. * @param {Db} db the database to query.
  995. * @param {string} [rootCollection] The root collection that holds the files and chunks collection. Defaults to **{GridStore.DEFAULT_ROOT_COLLECTION}**.
  996. * @param {object} [options=null] Optional settings.
  997. * @param {(ReadPreference|string)} [options.readPreference=null] The preferred read preference (ReadPreference.PRIMARY, ReadPreference.PRIMARY_PREFERRED, ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED, ReadPreference.NEAREST).
  998. * @param {GridStore~resultCallback} callback result from exists.
  999. * @return {null}
  1000. */
  1001. GridStore.list = function(db, rootCollection, options, callback) {
  1002. var args = Array.prototype.slice.call(arguments, 1);
  1003. callback = args.pop();
  1004. rootCollection = args.length ? args.shift() : null;
  1005. options = args.length ? args.shift() : {};
  1006. // Ensure we have correct values
  1007. if(rootCollection != null && typeof rootCollection == 'object') {
  1008. options = rootCollection;
  1009. rootCollection = null;
  1010. }
  1011. // Establish read preference
  1012. var readPreference = options.readPreference || ReadPreference.PRIMARY;
  1013. // Check if we are returning by id not filename
  1014. var byId = options['id'] != null ? options['id'] : false;
  1015. // Fetch item
  1016. var rootCollectionFinal = rootCollection != null ? rootCollection : GridStore.DEFAULT_ROOT_COLLECTION;
  1017. var items = [];
  1018. db.collection((rootCollectionFinal + ".files"), function(err, collection) {
  1019. if(err) return callback(err);
  1020. collection.find({}, {readPreference:readPreference}, function(err, cursor) {
  1021. if(err) return callback(err);
  1022. cursor.each(function(err, item) {
  1023. if(item != null) {
  1024. items.push(byId ? item._id : item.filename);
  1025. } else {
  1026. callback(err, items);
  1027. }
  1028. });
  1029. });
  1030. });
  1031. };
  1032. /**
  1033. * Reads the contents of a file.
  1034. *
  1035. * This method has the following signatures
  1036. *
  1037. * (db, name, callback)
  1038. * (db, name, length, callback)
  1039. * (db, name, length, offset, callback)
  1040. * (db, name, length, offset, options, callback)
  1041. *
  1042. * @method
  1043. * @static
  1044. * @param {Db} db the database to query.
  1045. * @param {string} name The name of the file.
  1046. * @param {number} [length] The size of data to read.
  1047. * @param {number} [offset] The offset from the head of the file of which to start reading from.
  1048. * @param {object} [options=null] Optional settings.
  1049. * @param {(ReadPreference|string)} [options.readPreference=null] The preferred read preference (ReadPreference.PRIMARY, ReadPreference.PRIMARY_PREFERRED, ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED, ReadPreference.NEAREST).
  1050. * @param {GridStore~readCallback} callback the command callback.
  1051. * @return {null}
  1052. */
  1053. GridStore.read = function(db, name, length, offset, options, callback) {
  1054. var args = Array.prototype.slice.call(arguments, 2);
  1055. callback = args.pop();
  1056. length = args.length ? args.shift() : null;
  1057. offset = args.length ? args.shift() : null;
  1058. options = args.length ? args.shift() : null;
  1059. new GridStore(db, name, "r", options).open(function(err, gridStore) {
  1060. if(err) return callback(err);
  1061. // Make sure we are not reading out of bounds
  1062. if(offset && offset >= gridStore.length) return callback("offset larger than size of file", null);
  1063. if(length && length > gridStore.length) return callback("length is larger than the size of the file", null);
  1064. if(offset && length && (offset + length) > gridStore.length) return callback("offset and length is larger than the size of the file", null);
  1065. if(offset != null) {
  1066. gridStore.seek(offset, function(err, gridStore) {
  1067. if(err) return callback(err);
  1068. gridStore.read(length, callback);
  1069. });
  1070. } else {
  1071. gridStore.read(length, callback);
  1072. }
  1073. });
  1074. };
  1075. /**
  1076. * Read the entire file as a list of strings splitting by the provided separator.
  1077. *
  1078. * @method
  1079. * @static
  1080. * @param {Db} db the database to query.
  1081. * @param {(String|object)} name the name of the file.
  1082. * @param {string} [separator] The character to be recognized as the newline separator.
  1083. * @param {object} [options=null] Optional settings.
  1084. * @param {(ReadPreference|string)} [options.readPreference=null] The preferred read preference (ReadPreference.PRIMARY, ReadPreference.PRIMARY_PREFERRED, ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED, ReadPreference.NEAREST).
  1085. * @param {GridStore~readlinesCallback} callback the command callback.
  1086. * @return {null}
  1087. */
  1088. GridStore.readlines = function(db, name, separator, options, callback) {
  1089. var args = Array.prototype.slice.call(arguments, 2);
  1090. callback = args.pop();
  1091. separator = args.length ? args.shift() : null;
  1092. options = args.length ? args.shift() : null;
  1093. var finalSeperator = separator == null ? "\n" : separator;
  1094. new GridStore(db, name, "r", options).open(function(err, gridStore) {
  1095. if(err) return callback(err);
  1096. gridStore.readlines(finalSeperator, callback);
  1097. });
  1098. };
  1099. /**
  1100. * Deletes the chunks and metadata information of a file from GridFS.
  1101. *
  1102. * @method
  1103. * @static
  1104. * @param {Db} db The database to query.
  1105. * @param {(string|array)} names The name/names of the files to delete.
  1106. * @param {object} [options=null] Optional settings.
  1107. * @param {GridStore~resultCallback} callback the command callback.
  1108. * @return {null}
  1109. */
  1110. GridStore.unlink = function(db, names, options, callback) {
  1111. var self = this;
  1112. var args = Array.prototype.slice.call(arguments, 2);
  1113. callback = args.pop();
  1114. options = args.length ? args.shift() : {};
  1115. // Get the write concern
  1116. var writeConcern = _getWriteConcern(db, options);
  1117. // List of names
  1118. if(names.constructor == Array) {
  1119. var tc = 0;
  1120. for(var i = 0; i < names.length; i++) {
  1121. ++tc;
  1122. GridStore.unlink(db, names[i], options, function(result) {
  1123. if(--tc == 0) {
  1124. callback(null, self);
  1125. }
  1126. });
  1127. }
  1128. } else {
  1129. new GridStore(db, names, "w", options).open(function(err, gridStore) {
  1130. if(err) return callback(err);
  1131. deleteChunks(gridStore, function(err, result) {
  1132. if(err) return callback(err);
  1133. gridStore.collection(function(err, collection) {
  1134. if(err) return callback(err);
  1135. collection.remove({'_id':gridStore.fileId}, writeConcern, function(err, result) {
  1136. callback(err, self);
  1137. });
  1138. });
  1139. });
  1140. });
  1141. }
  1142. };
  1143. /**
  1144. * @ignore
  1145. */
  1146. var _writeNormal = function(self, data, close, callback) {
  1147. // If we have a buffer write it using the writeBuffer method
  1148. if(Buffer.isBuffer(data)) {
  1149. return writeBuffer(self, data, close, callback);
  1150. } else {
  1151. return writeBuffer(self, new Buffer(data, 'binary'), close, callback);
  1152. }
  1153. }
  1154. /**
  1155. * @ignore
  1156. */
  1157. var _setWriteConcernHash = function(options) {
  1158. var finalOptions = {};
  1159. if(options.w != null) finalOptions.w = options.w;
  1160. if(options.journal == true) finalOptions.j = options.journal;
  1161. if(options.j == true) finalOptions.j = options.j;
  1162. if(options.fsync == true) finalOptions.fsync = options.fsync;
  1163. if(options.wtimeout != null) finalOptions.wtimeout = options.wtimeout;
  1164. return finalOptions;
  1165. }
  1166. /**
  1167. * @ignore
  1168. */
  1169. var _getWriteConcern = function(self, options) {
  1170. // Final options
  1171. var finalOptions = {w:1};
  1172. options = options || {};
  1173. // Local options verification
  1174. if(options.w != null || typeof options.j == 'boolean' || typeof options.journal == 'boolean' || typeof options.fsync == 'boolean') {
  1175. finalOptions = _setWriteConcernHash(options);
  1176. } else if(options.safe != null && typeof options.safe == 'object') {
  1177. finalOptions = _setWriteConcernHash(options.safe);
  1178. } else if(typeof options.safe == "boolean") {
  1179. finalOptions = {w: (options.safe ? 1 : 0)};
  1180. } else if(self.options.w != null || typeof self.options.j == 'boolean' || typeof self.options.journal == 'boolean' || typeof self.options.fsync == 'boolean') {
  1181. finalOptions = _setWriteConcernHash(self.options);
  1182. } else if(self.safe && (self.safe.w != null || typeof self.safe.j == 'boolean' || typeof self.safe.journal == 'boolean' || typeof self.safe.fsync == 'boolean')) {
  1183. finalOptions = _setWriteConcernHash(self.safe);
  1184. } else if(typeof self.safe == "boolean") {
  1185. finalOptions = {w: (self.safe ? 1 : 0)};
  1186. }
  1187. // Ensure we don't have an invalid combination of write concerns
  1188. if(finalOptions.w < 1
  1189. && (finalOptions.journal == true || finalOptions.j == true || finalOptions.fsync == true)) throw new MongoError("No acknowledgement using w < 1 cannot be combined with journal:true or fsync:true");
  1190. // Return the options
  1191. return finalOptions;
  1192. }
  1193. /**
  1194. * Create a new GridStoreStream instance (INTERNAL TYPE, do not instantiate directly)
  1195. *
  1196. * @class
  1197. * @extends external:Duplex
  1198. * @return {GridStoreStream} a GridStoreStream instance.
  1199. */
  1200. var GridStoreStream = function(gs) {
  1201. var self = this;
  1202. // Initialize the duplex stream
  1203. Duplex.call(this);
  1204. // Get the gridstore
  1205. this.gs = gs;
  1206. // End called
  1207. this.endCalled = false;
  1208. // If we have a seek
  1209. this.totalBytesToRead = this.gs.length - this.gs.position;
  1210. this.seekPosition = this.gs.position;
  1211. }
  1212. //
  1213. // Inherit duplex
  1214. inherits(GridStoreStream, Duplex);
  1215. GridStoreStream.prototype._pipe = GridStoreStream.prototype.pipe;
  1216. // Set up override
  1217. GridStoreStream.prototype.pipe = function(destination) {
  1218. var self = this;
  1219. // Only open gridstore if not already open
  1220. if(!self.gs.isOpen) {
  1221. self.gs.open(function(err) {
  1222. if(err) return self.emit('error', err);
  1223. self.totalBytesToRead = self.gs.length - self.gs.position;
  1224. self._pipe.apply(self, [destination]);
  1225. });
  1226. } else {
  1227. self.totalBytesToRead = self.gs.length - self.gs.position;
  1228. self._pipe.apply(self, [destination]);
  1229. }
  1230. }
  1231. // Called by stream
  1232. GridStoreStream.prototype._read = function(n) {
  1233. var self = this;
  1234. var read = function() {
  1235. // Read data
  1236. self.gs.read(length, function(err, buffer) {
  1237. if(err && !self.endCalled) return self.emit('error', err);
  1238. // Stream is closed
  1239. if(self.endCalled || buffer == null) return self.push(null);
  1240. // Remove bytes read
  1241. if(buffer.length <= self.totalBytesToRead) {
  1242. self.totalBytesToRead = self.totalBytesToRead - buffer.length;
  1243. self.push(buffer);
  1244. } else if(buffer.length > self.totalBytesToRead) {
  1245. self.totalBytesToRead = self.totalBytesToRead - buffer._index;
  1246. self.push(buffer.slice(0, buffer._index));
  1247. }
  1248. // Finished reading
  1249. if(self.totalBytesToRead <= 0) {
  1250. self.endCalled = true;
  1251. }
  1252. });
  1253. }
  1254. // Set read length
  1255. var length = self.gs.length < self.gs.chunkSize ? self.gs.length - self.seekPosition : self.gs.chunkSize;
  1256. if(!self.gs.isOpen) {
  1257. self.gs.open(function(err, gs) {
  1258. self.totalBytesToRead = self.gs.length - self.gs.position;
  1259. if(err) return self.emit('error', err);
  1260. read();
  1261. });
  1262. } else {
  1263. read();
  1264. }
  1265. }
  1266. GridStoreStream.prototype.destroy = function() {
  1267. this.pause();
  1268. this.endCalled = true;
  1269. this.gs.close();
  1270. this.emit('end');
  1271. }
  1272. GridStoreStream.prototype.write = function(chunk, encoding, callback) {
  1273. var self = this;
  1274. if(self.endCalled) return self.emit('error', new MongoError('attempting to write to stream after end called'))
  1275. // Do we have to open the gridstore
  1276. if(!self.gs.isOpen) {
  1277. self.gs.open(function() {
  1278. self.gs.isOpen = true;
  1279. self.gs.write(chunk, function() {
  1280. process.nextTick(function() {
  1281. self.emit('drain');
  1282. });
  1283. });
  1284. });
  1285. return false;
  1286. } else {
  1287. self.gs.write(chunk, function() {
  1288. self.emit('drain');
  1289. });
  1290. return true;
  1291. }
  1292. }
  1293. GridStoreStream.prototype.end = function(chunk, encoding, callback) {
  1294. var self = this;
  1295. var args = Array.prototype.slice.call(arguments, 0);
  1296. callback = args.pop();
  1297. chunk = args.length ? args.shift() : null;
  1298. encoding = args.length ? args.shift() : null;
  1299. self.endCalled = true;
  1300. if(chunk) {
  1301. self.gs.write(chunk, function() {
  1302. self.gs.close(function() {
  1303. if(typeof callback == 'function') callback();
  1304. self.emit('end')
  1305. });
  1306. });
  1307. }
  1308. self.gs.close(function() {
  1309. if(typeof callback == 'function') callback();
  1310. self.emit('end')
  1311. });
  1312. }
  1313. /**
  1314. * The read() method pulls some data out of the internal buffer and returns it. If there is no data available, then it will return null.
  1315. * @function external:Duplex#read
  1316. * @param {number} size Optional argument to specify how much data to read.
  1317. * @return {(String | Buffer | null)}
  1318. */
  1319. /**
  1320. * Call this function to cause the stream to return strings of the specified encoding instead of Buffer objects.
  1321. * @function external:Duplex#setEncoding
  1322. * @param {string} encoding The encoding to use.
  1323. * @return {null}
  1324. */
  1325. /**
  1326. * This method will cause the readable stream to resume emitting data events.
  1327. * @function external:Duplex#resume
  1328. * @return {null}
  1329. */
  1330. /**
  1331. * This method will cause a stream in flowing-mode to stop emitting data events. Any data that becomes available will remain in the internal buffer.
  1332. * @function external:Duplex#pause
  1333. * @return {null}
  1334. */
  1335. /**
  1336. * This method pulls all the data out of a readable stream, and writes it to the supplied destination, automatically managing the flow so that the destination is not overwhelmed by a fast readable stream.
  1337. * @function external:Duplex#pipe
  1338. * @param {Writable} destination The destination for writing data
  1339. * @param {object} [options] Pipe options
  1340. * @return {null}
  1341. */
  1342. /**
  1343. * This method will remove the hooks set up for a previous pipe() call.
  1344. * @function external:Duplex#unpipe
  1345. * @param {Writable} [destination] The destination for writing data
  1346. * @return {null}
  1347. */
  1348. /**
  1349. * This is useful in certain cases where a stream is being consumed by a parser, which needs to "un-consume" some data that it has optimistically pulled out of the source, so that the stream can be passed on to some other party.
  1350. * @function external:Duplex#unshift
  1351. * @param {(Buffer|string)} chunk Chunk of data to unshift onto the read queue.
  1352. * @return {null}
  1353. */
  1354. /**
  1355. * Versions of Node prior to v0.10 had streams that did not implement the entire Streams API as it is today. (See "Compatibility" below for more information.)
  1356. * @function external:Duplex#wrap
  1357. * @param {Stream} stream An "old style" readable stream.
  1358. * @return {null}
  1359. */
  1360. /**
  1361. * This method writes some data to the underlying system, and calls the supplied callback once the data has been fully handled.
  1362. * @function external:Duplex#write
  1363. * @param {(string|Buffer)} chunk The data to write
  1364. * @param {string} encoding The encoding, if chunk is a String
  1365. * @param {function} callback Callback for when this chunk of data is flushed
  1366. * @return {boolean}
  1367. */
  1368. /**
  1369. * Call this method when no more data will be written to the stream. If supplied, the callback is attached as a listener on the finish event.
  1370. * @function external:Duplex#end
  1371. * @param {(string|Buffer)} chunk The data to write
  1372. * @param {string} encoding The encoding, if chunk is a String
  1373. * @param {function} callback Callback for when this chunk of data is flushed
  1374. * @return {null}
  1375. */
  1376. /**
  1377. * GridStoreStream stream data event, fired for each document in the cursor.
  1378. *
  1379. * @event GridStoreStream#data
  1380. * @type {object}
  1381. */
  1382. /**
  1383. * GridStoreStream stream end event
  1384. *
  1385. * @event GridStoreStream#end
  1386. * @type {null}
  1387. */
  1388. /**
  1389. * GridStoreStream stream close event
  1390. *
  1391. * @event GridStoreStream#close
  1392. * @type {null}
  1393. */
  1394. /**
  1395. * GridStoreStream stream readable event
  1396. *
  1397. * @event GridStoreStream#readable
  1398. * @type {null}
  1399. */
  1400. /**
  1401. * GridStoreStream stream drain event
  1402. *
  1403. * @event GridStoreStream#drain
  1404. * @type {null}
  1405. */
  1406. /**
  1407. * GridStoreStream stream finish event
  1408. *
  1409. * @event GridStoreStream#finish
  1410. * @type {null}
  1411. */
  1412. /**
  1413. * GridStoreStream stream pipe event
  1414. *
  1415. * @event GridStoreStream#pipe
  1416. * @type {null}
  1417. */
  1418. /**
  1419. * GridStoreStream stream unpipe event
  1420. *
  1421. * @event GridStoreStream#unpipe
  1422. * @type {null}
  1423. */
  1424. /**
  1425. * GridStoreStream stream error event
  1426. *
  1427. * @event GridStoreStream#error
  1428. * @type {null}
  1429. */
  1430. /**
  1431. * @ignore
  1432. */
  1433. module.exports = GridStore;