add dried out version of tempFileHandler as option for saving files

with test. passes all tests
6 years ago · e5b77f10e9
parent 83f11ba06f
commit e5b77f10e9
7 changed files with 205 additions and 30 deletions
--- a/README.md
+++ b/README.md
@ -53,6 +53,16 @@ app.use(fileUpload({
 }));
 ```

+### Using useTempFile Options
+Use temp files instead of memory for managing the upload process.
+Please note: md5 hashes will not be generated when using tempFiles
+```javascript
+app.use(fileUpload({
+    useTempFiles : true,
+    tempFileDir : '/tmp/'
+}));
+```
+
 ### Available Options
 Pass in non-Busboy options directly to the middleware. These are express-fileupload specific options.

@ -62,6 +72,8 @@ createParentPath | <ul><li><code>false</code>&nbsp;**(default)**</li><li><code>t
 safeFileNames | <ul><li><code>false</code>&nbsp;**(default)**</li><li><code>true</code></li><li>regex</li></ul> | Strips characters from the upload's filename. You can use custom regex to determine what to strip. If set to `true`, non-alphanumeric characters _except_ dashes and underscores will be stripped. This option is off by default.<br /><br />**Example #1 (strip slashes from file names):** `app.use(fileUpload({ safeFileNames: /\\/g }))`<br />**Example #2:** `app.use(fileUpload({ safeFileNames: true }))`
 preserveExtension | <ul><li><code>false</code>&nbsp;**(default)**</li><li><code>true</code></li><li><code>*Number*</code></li></ul> | Preserves filename extension when using <code>safeFileNames</code> option. If set to <code>true</code>, will default to an extension length of 3. If set to <code>*Number*</code>, this will be the max allowable extension length. If an extension is smaller than the extension length, it remains untouched. If the extension is longer, it is shifted.<br /><br />**Example #1 (true):**<br /><code>app.use(fileUpload({ safeFileNames: true, preserveExtension: true }));</code><br />*myFileName.ext* --> *myFileName.ext*<br /><br />**Example #2 (max extension length 2, extension shifted):**<br /><code>app.use(fileUpload({ safeFileNames: true, preserveExtension: 2 }));</code><br />*myFileName.ext* --> *myFileNamee.xt*
 abortOnLimit | <ul><li><code>false</code>&nbsp;**(default)**</li><li><code>true</code></ul> | Returns a HTTP 413 when the file is bigger than the size limit if true. Otherwise, it will add a <code>truncate = true</code> to the resulting file structure.
+useTempFiles | <ul><li><code>false</code>&nbsp;**(default)**</li><li><code>true</code></ul> | Will use temporary files at the specified tempDir for managing uploads rather than using buffers in memory. This avoids memory issues when uploading large files.
+tempFileDir | <ul><li><code>String</code>&nbsp;**(path)**</li></ul> | Used with the <code>useTempFiles</code> option. Path to the directory where temp files will be stored during the upload process. Add trailing slash.

 # Help Wanted
 Looking for additional maintainers. Please contact `richardgirges [ at ] gmail.com` if you're interested. Pull Requests are welcomed! 
--- a/example/index.html
+++ b/example/index.html
@ -2,11 +2,11 @@
 	<body>
 		<form ref='uploadForm' 
 			id='uploadForm' 
-			action='http://localhost:8000/upload' 
+			action='/upload' 
 			method='post' 
 			encType="multipart/form-data">
 				<input type="file" name="sampleFile" />
 				<input type='submit' value='Upload!' />
 		</form>		
 	</body>
-</html>
+</html>
--- a/example/server.js
+++ b/example/server.js
@ -1,40 +1,40 @@
-const express = require('express');
-const fileUpload = require('../lib/index.js');
+const express = require("express");
+const fileUpload = require("express-fileupload");
 const app = express();

-app.use('/form', express.static(__dirname + '/index.html'));
+app.use("/form", express.static(__dirname + "/index.html"));

 // default options
 app.use(fileUpload());

-app.get('/ping', function(req, res) {
-  res.send('pong');
+app.get("/ping", function(req, res) {
+  res.send("pong");
 });

-app.post('/upload', function(req, res) {
+app.post("/upload", function(req, res) {
  let sampleFile;
  let uploadPath;

  if (Object.keys(req.files).length == 0) {
-    res.status(400).send('No files were uploaded.');
+    res.status(400).send("No files were uploaded.");
    return;
  }

-  console.log('req.files >>>', req.files); // eslint-disable-line
+  console.log("req.files >>>", req.files); // eslint-disable-line

  sampleFile = req.files.sampleFile;

-  uploadPath = __dirname + '/uploads/' + sampleFile.name;
+  uploadPath = __dirname + "/uploads/" + sampleFile.name;

  sampleFile.mv(uploadPath, function(err) {
    if (err) {
      return res.status(500).send(err);
    }

-    res.send('File uploaded to ' + uploadPath);
+    res.send("File uploaded to " + uploadPath);
  });
 });

-app.listen(8000, function() {
-  console.log('Express server listening on port 8000'); // eslint-disable-line
+app.listen(8002, function() {
+  console.log("Express server listening on port 8002"); // eslint-disable-line
 });
--- a/lib/fileFactory.js
+++ b/lib/fileFactory.js
@ -6,7 +6,7 @@ const streamifier = require('streamifier');
 const md5 = require('md5');

 module.exports = function(options, fileUploadOptions = null) {
-  return {
+  const output = {
    name: options.name,
    data: options.buffer,
    encoding: options.encoding,
@ -16,19 +16,34 @@ module.exports = function(options, fileUploadOptions = null) {
    mv: function(filePath, callback) {
      // Callback is passed in, use the callback API
      if (callback) {
-        doMove(
-          () => {
-            callback(null);
-          },
+        if(!options.buffer){
+          doMoveFromTemp(
+            () => {
+              callback(null);
+            },
+            (error) => {
+              callback(error);
+            }
+          );
+        } else {
+          doMoveFromBuffer(
+            () => {
+              callback(null);
+            },
          (error) => {
            callback(error);
          }
        );
+}

      // Otherwise, return a promise
      } else {
        return new Promise((resolve, reject) => {
-          doMove(resolve, reject);
+          if(options.buffer) {
+            doMoveFromBuffer(resolve, reject);
+          } else {
+            doMoveFromTemp(resolve, reject);
+          }
        });
      }

@ -38,7 +53,16 @@ module.exports = function(options, fileUploadOptions = null) {
       * @param {Function} successFunc
       * @param {Function} errorFunc
       */
-      function doMove(successFunc, errorFunc) {
+      function doMoveFromTemp(successFunc, errorFunc){
+        fs.rename(options.tempFilePath, path, function(err){
+          if(err) {
+            errorFunc(err);
+          } else {
+            successFunc();
+          }
+        })
+      }
+      function doMoveFromBuffer(successFunc, errorFunc) {
        if (fileUploadOptions && fileUploadOptions.createParentPath) {
          const parentPath = path.dirname(filePath);
          if (!fs.existsSync(parentPath)) {
@ -60,4 +84,8 @@ module.exports = function(options, fileUploadOptions = null) {
      }
    }
  };
+  if(options.size){
+    output.size = options.size
+  }
+  return output;
 };
--- a/lib/processMultipart.js
+++ b/lib/processMultipart.js
@ -1,5 +1,6 @@
 const Busboy = require('busboy');
 const fileFactory = require('./fileFactory');
+const {cleanupStream, tempFileHandler} = require('./tempFileHandler')

 /**
 * Processes multipart request
@ -51,6 +52,15 @@ module.exports = function processMultipart(options, req, res, next) {
  busboy.on('file', function(fieldname, file, filename, encoding, mime) {
    const buffers = [];
    let safeFileNameRegex = /[^\w-]/g;
+    const memHandler = function(data) {
+      buffers.push(data);
+      if (options.debug) {
+        return console.log('Uploading %s -> %s', fieldname, filename); // eslint-disable-line
+      }
+    }
+    const dataHandler = options.useTempFiles ? tempFileHandler(options, fieldname, filename) : memHandler;
+    const memoryUploadHandler = function(fieldname, file, filename, encoding, mime) {
+    }

    file.on('limit', () => {
      if (options.abortOnLimit) {
@ -59,18 +69,15 @@ module.exports = function processMultipart(options, req, res, next) {
      }
    });

-    file.on('data', function(data) {
-      buffers.push(data);
-
-      if (options.debug) {
-        return console.log('Uploading %s -> %s', fieldname, filename); // eslint-disable-line
-      }
-    });
+    file.on('data', memHandler);

    file.on('end', function() {
      if (!req.files) {
        req.files = {};
      }
+      if(options.useTempFiles) {
+        cleanupStream();
+      }

      const buf = Buffer.concat(buffers);
      // see: https://github.com/richardgirges/express-fileupload/issues/14
@ -136,7 +143,7 @@ module.exports = function processMultipart(options, req, res, next) {
      }
    });

-    file.on('error', next);
+    file.on('error', cleanupStream, next);
  });

  busboy.on('finish', next);
@ -144,4 +151,4 @@ module.exports = function processMultipart(options, req, res, next) {
  busboy.on('error', next);

  req.pipe(busboy);
-};
+};
--- a/lib/tempFileHandler.js
+++ b/lib/tempFileHandler.js
@ -0,0 +1,29 @@
+const fs = require("fs");
+let writeStream;
+let tempFilePath;
+
+module.exports.cleanupStream = function() {
+  writeStream.end();
+  fs.unlink(tempFilePath, function(err) {
+    if (err) throw err;
+  });
+};
+
+module.exports.tempFileHandler = function(options, fieldname, filename) {
+  const dir = options.tempFileDir || "/tmp/";
+  tempFilePath = dir + "tmp" + Date.now();
+  writeStream = fs.createWriteStream(tempFilePath);
+  let fileSize = 0;
+
+  return function(data) {
+    writeStream.write(data);
+    fileSize += data.length;
+    if (options.debug) {
+      return console.log(
+        `Uploaded ${data.length} bytes for `,
+        fieldname,
+        filename
+      );
+    }
+  };
+};
--- a/test/tempFile.spec.js
+++ b/test/tempFile.spec.js
@ -0,0 +1,99 @@
+const fs = require("fs");
+const path = require("path");
+const request = require("supertest");
+const server = require("./server");
+const clearUploadsDir = server.clearUploadsDir;
+const fileDir = server.fileDir;
+const uploadDir = server.uploadDir;
+describe("File Upload Options Tests", function() {
+  afterEach(function(done) {
+    clearUploadsDir();
+    done();
+  });
+  /**
+   * Upload the file for testing and verify the expected filename.
+   * @param {object} options The expressFileUpload options.
+   * @param {string} actualFileNameToUpload The name of the file to upload.
+   * @param {string} expectedFileNameOnFileSystem The name of the file after upload.
+   * @param {function} done The mocha continuation function.
+   */
+  function executeFileUploadTestWalk(
+    options,
+    actualFileNameToUpload,
+    expectedFileNameOnFileSystem,
+    done
+  ) {
+    request(server.setup(options))
+      .post("/upload/single")
+      .attach("testFile", path.join(fileDir, actualFileNameToUpload))
+      .expect(200)
+      .end(function(err) {
+        if (err) {
+          return done(err);
+        }
+        const uploadedFilePath = path.join(
+          uploadDir,
+          expectedFileNameOnFileSystem
+        );
+        fs.stat(uploadedFilePath, done);
+      });
+  }
+  describe("Testing [safeFileNames with useTempFiles] option to ensure:", function() {
+    it("Does nothing to your filename when disabled.", function(done) {
+      const fileUploadOptions = {
+        safeFileNames: false,
+        useTempFiles: true,
+        tempFileDir: "/tmp/"
+      };
+      const actualFileName = "my$Invalid#fileName.png123";
+      const expectedFileName = "my$Invalid#fileName.png123";
+      executeFileUploadTestWalk(
+        fileUploadOptions,
+        actualFileName,
+        expectedFileName,
+        done
+      );
+    });
+    it("Is disabled by default.", function(done) {
+      const fileUploadOptions = { useTempFiles: true, tempFileDir: "/tmp/" };
+      const actualFileName = "my$Invalid#fileName.png123";
+      const expectedFileName = "my$Invalid#fileName.png123";
+      executeFileUploadTestWalk(
+        fileUploadOptions,
+        actualFileName,
+        expectedFileName,
+        done
+      );
+    });
+    it("Strips away all non-alphanumeric characters (excluding hyphens/underscores) when enabled.", function(done) {
+      const fileUploadOptions = {
+        safeFileNames: true,
+        useTempFiles: true,
+        tempFileDir: "/tmp/"
+      };
+      const actualFileName = "my$Invalid#fileName.png123";
+      const expectedFileName = "myInvalidfileNamepng123";
+      executeFileUploadTestWalk(
+        fileUploadOptions,
+        actualFileName,
+        expectedFileName,
+        done
+      );
+    });
+    it('Accepts a regex for stripping (decidedly) "invalid" characters from filename.', function(done) {
+      const fileUploadOptions = {
+        safeFileNames: /[\$#]/g,
+        useTempFiles: true,
+        tempFileDir: "/tmp/"
+      };
+      const actualFileName = "my$Invalid#fileName.png123";
+      const expectedFileName = "myInvalidfileName.png123";
+      executeFileUploadTestWalk(
+        fileUploadOptions,
+        actualFileName,
+        expectedFileName,
+        done
+      );
+    });
+  });
+});