From 64aedcaa6b3e9170e77c428ee306830464b42bcf Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 14 Nov 2017 03:46:30 +0900 Subject: [PATCH] =?UTF-8?q?add-file-to-drive=20-=20Promise=E7=99=BE?= =?UTF-8?q?=E7=83=88=E6=8B=B3=E3=81=A8=E3=83=A1=E3=83=A2=E3=83=AA=E5=89=8A?= =?UTF-8?q?=E6=B8=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 9 +- src/api/common/add-file-to-drive.ts | 338 +++++++++++------- .../endpoints/drive/files/upload_from_url.ts | 74 +++- 3 files changed, 269 insertions(+), 152 deletions(-) diff --git a/package.json b/package.json index 5b26ee574..235e64f7c 100644 --- a/package.json +++ b/package.json @@ -54,13 +54,14 @@ "@types/node": "8.0.49", "@types/page": "1.5.32", "@types/proxy-addr": "2.0.0", - "@types/seedrandom": "2.4.27", "@types/ratelimiter": "2.1.28", "@types/redis": "2.8.1", - "@types/request": "2.0.7", + "@types/request": "^2.0.7", "@types/rimraf": "2.0.2", "@types/riot": "3.6.1", + "@types/seedrandom": "2.4.27", "@types/serve-favicon": "2.2.29", + "@types/tmp": "0.0.33", "@types/uuid": "3.4.3", "@types/webpack": "3.8.0", "@types/webpack-stream": "3.2.8", @@ -111,7 +112,6 @@ "deep-equal": "1.0.1", "deepcopy": "0.6.3", "diskusage": "0.2.2", - "download": "6.2.5", "elasticsearch": "13.3.1", "escape-regexp": "0.0.1", "express": "4.15.4", @@ -140,7 +140,7 @@ "recaptcha-promise": "0.1.3", "reconnecting-websocket": "3.2.2", "redis": "2.8.0", - "request": "2.83.0", + "request": "^2.83.0", "rimraf": "2.6.2", "riot": "3.7.4", "rndstr": "1.0.0", @@ -152,6 +152,7 @@ "syuilo-password-strength": "0.0.1", "tcp-port-used": "0.1.2", "textarea-caret": "3.0.2", + "tmp": "0.0.33", "ts-node": "3.3.0", "typescript": "2.6.1", "uuid": "3.1.0", diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index a96906d29..a7c7cb464 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -9,28 +9,34 @@ import DriveFolder from '../models/drive-folder'; import serialize from '../serializers/drive-file'; import event from '../event'; import config from '../../conf'; -import { Duplex } from 'stream'; +import { Buffer } from 'buffer'; +import * as fs from 'fs'; +import * as tmp from 'tmp'; +import * as stream from 'stream'; const log = debug('misskey:register-drive-file'); -const addToGridFS = (name, binary, type, metadata): Promise => new Promise(async (resolve, reject) => { - const dataStream = new Duplex(); - dataStream.push(binary); - dataStream.push(null); +const tmpFile = (): Promise => new Promise((resolve, reject) => { + tmp.file((e, path) => { + if (e) return reject(e) + resolve(path) + }) +}) - const bucket = await getGridFSBucket(); - const writeStream = bucket.openUploadStream(name, { contentType: type, metadata }); - writeStream.once('finish', (doc) => { resolve(doc); }); - writeStream.on('error', reject); - dataStream.pipe(writeStream); -}); +const addToGridFS = (name: string, readable: stream.Readable, type: string, metadata: any): Promise => + getGridFSBucket() + .then(bucket => new Promise((resolve, reject) => { + const writeStream = bucket.openUploadStream(name, { contentType: type, metadata }); + writeStream.once('finish', (doc) => { resolve(doc); }); + writeStream.on('error', reject); + readable.pipe(writeStream); + })) /** * Add file to drive * * @param user User who wish to add file - * @param fileName File name - * @param data Contents + * @param file File path, binary, or readableStream * @param comment Comment * @param type File type * @param folderId Folder ID @@ -39,139 +45,201 @@ const addToGridFS = (name, binary, type, metadata): Promise => new Promise( */ export default ( user: any, - data: Buffer, + file: string | Buffer | stream.Readable, name: string = null, comment: string = null, folderId: mongodb.ObjectID = null, force: boolean = false -) => new Promise(async (resolve, reject) => { +) => new Promise((resolve, reject) => { log(`registering ${name} (user: ${user.username})`); - // File size - const size = data.byteLength; - - log(`size is ${size}`); - - // File type - let mime = 'application/octet-stream'; - const type = fileType(data); - if (type !== null) { - mime = type.mime; - - if (name === null) { - name = `untitled.${type.ext}`; + // Get file path + new Promise((res: (v: string) => void, rej) => { + if (typeof file === 'string') { + res(file) + return } - } else { - if (name === null) { - name = 'untitled'; + if (file instanceof Buffer) { + tmpFile() + .then(path => { + fs.writeFile(path, file, (err) => { + if (err) rej(err) + res(path) + }) + }) + .catch(rej) + return } - } - - log(`type is ${mime}`); - - // Generate hash - const hash = crypto - .createHash('md5') - .update(data) - .digest('hex') as string; - - log(`hash is ${hash}`); - - if (!force) { - // Check if there is a file with the same hash - const much = await DriveFile.findOne({ - md5: hash, - 'metadata.user_id': user._id - }); - - if (much !== null) { - log('file with same hash is found'); - return resolve(much); - } else { - log('file with same hash is not found'); + if (typeof file === 'object' && typeof file.read === 'function') { + tmpFile() + .then(path => { + const readable: stream.Readable = file + const writable = fs.createWriteStream(path) + readable + .on('error', rej) + .on('end', () => { + res(path) + }) + .pipe(writable) + .on('error', rej) + }) + .catch(rej) } - } + rej(new Error('un-compatible file.')) + }) + // Calculate hash, get content type and get file size + .then(path => Promise.all([ + path, + // hash + ((): Promise => new Promise((res, rej) => { + const readable = fs.createReadStream(path) + const hash = crypto.createHash('md5') + readable + .on('error', rej) + .on('end', () => { + res(hash.digest('hex')) + }) + .pipe(hash) + .on('error', rej) + }))(), + // mime + ((): Promise<[string, string | null]> => new Promise((res, rej) => { + const readable = fs.createReadStream(path) + readable + .on('error', rej) + .once('data', (buffer: Buffer) => { + readable.destroy() + const type = fileType(buffer) + if (!type) { + return res(['application/octet-stream', null]) + } + return res([type.mime, type.ext]) + }) + }))(), + // size + ((): Promise => new Promise((res, rej) => { + fs.stat(path, (err, stats) => { + if (err) return rej(err) + res(stats.size) + }) + }))() + ])) + .then(async ([path, hash, [mime, ext], size]) => { + log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`) - // Calculate drive usage - const usage = ((await DriveFile - .aggregate([ - { $match: { 'metadata.user_id': user._id } }, - { $project: { - length: true - }}, - { $group: { - _id: null, - usage: { $sum: '$length' } - }} - ]))[0] || { - usage: 0 - }).usage; + // detect name + const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled'); - log(`drive usage is ${usage}`); + if (!force) { + // Check if there is a file with the same hash + const much = await DriveFile.findOne({ + md5: hash, + 'metadata.user_id': user._id + }); - // If usage limit exceeded - if (usage + size > user.drive_capacity) { - return reject('no-free-space'); - } - - // If the folder is specified - let folder: any = null; - if (folderId !== null) { - folder = await DriveFolder - .findOne({ - _id: folderId, - user_id: user._id - }); - - if (folder === null) { - return reject('folder-not-found'); - } - } - - let properties: any = null; - - // If the file is an image - if (/^image\/.*$/.test(mime)) { - // Calculate width and height to save in property - const g = gm(data, name); - const size = await prominence(g).size(); - properties = { - width: size.width, - height: size.height - }; - - log('image width and height is calculated'); - } - - // Create DriveFile document - const file = await addToGridFS(name, data, mime, { - user_id: user._id, - folder_id: folder !== null ? folder._id : null, - comment: comment, - properties: properties - }); - - log(`drive file has been created ${file._id}`); - - resolve(file); - - // Serialize - const fileObj = await serialize(file); - - // Publish drive_file_created event - event(user._id, 'drive_file_created', fileObj); - - // Register to search database - if (config.elasticsearch.enable) { - const es = require('../../db/elasticsearch'); - es.index({ - index: 'misskey', - type: 'drive_file', - id: file._id.toString(), - body: { - name: file.name, - user_id: user._id.toString() + if (much !== null) { + log('file with same hash is found'); + return resolve(much); + } else { + log('file with same hash is not found'); + } } - }); - } + + const [properties, folder] = await Promise.all([ + // properties + (async () => { + if (!/^image\/.*$/.test(mime)) { + return null + } + // If the file is an image, calculate width and height to save in property + const g = gm(data, name); + const size = await prominence(g).size(); + const properties = { + width: size.width, + height: size.height + }; + log('image width and height is calculated'); + return properties + })(), + // folder + (async () => { + if (!folderId) { + return null + } + const driveFolder = await DriveFolder.findOne({ + _id: folderId, + user_id: user._id + }) + if (!driveFolder) { + throw 'folder-not-found' + } + return driveFolder + })(), + // usage checker + (async () => { + // Calculate drive usage + const usage = await DriveFile + .aggregate([ + { $match: { 'metadata.user_id': user._id } }, + { + $project: { + length: true + } + }, + { + $group: { + _id: null, + usage: { $sum: '$length' } + } + } + ]) + .then((aggregates: any[]) => { + if (aggregates.length > 0) { + return aggregates[0].usage + } + return 0 + }); + + log(`drive usage is ${usage}`); + + // If usage limit exceeded + if (usage + size > user.drive_capacity) { + throw 'no-free-space'; + } + })() + ]) + + const readable = fs.createReadStream(path) + + return addToGridFS(name, readable, mime, { + user_id: user._id, + folder_id: folder !== null ? folder._id : null, + comment: comment, + properties: properties + }) + }) + .then(file => { + log(`drive file has been created ${file._id}`); + resolve(file) + return serialize(file) + }) + .then(serializedFile => { + // Publish drive_file_created event + event(user._id, 'drive_file_created', fileObj); + + // Register to search database + if (config.elasticsearch.enable) { + const es = require('../../db/elasticsearch'); + es.index({ + index: 'misskey', + type: 'drive_file', + id: file._id.toString(), + body: { + name: file.name, + user_id: user._id.toString() + } + }); + } + }) + .catch(reject) }); diff --git a/src/api/endpoints/drive/files/upload_from_url.ts b/src/api/endpoints/drive/files/upload_from_url.ts index 46cfffb69..9c759994e 100644 --- a/src/api/endpoints/drive/files/upload_from_url.ts +++ b/src/api/endpoints/drive/files/upload_from_url.ts @@ -2,11 +2,17 @@ * Module dependencies */ import * as URL from 'url'; -const download = require('download'); import $ from 'cafy'; import { validateFileName } from '../../../models/drive-file'; import serialize from '../../../serializers/drive-file'; import create from '../../../common/add-file-to-drive'; +import * as debug from 'debug'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as request from 'request'; +import * as crypto from 'crypto'; + +const log = debug('misskey:endpoint:upload_from_url') /** * Create a file from a URL @@ -15,7 +21,7 @@ import create from '../../../common/add-file-to-drive'; * @param {any} user * @return {Promise} */ -module.exports = (params, user) => new Promise(async (res, rej) => { +module.exports = (params, user) => new Promise((res, rej) => { // Get 'url' parameter // TODO: Validate this url const [url, urlErr] = $(params.url).string().$; @@ -30,15 +36,57 @@ module.exports = (params, user) => new Promise(async (res, rej) => { const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; if (folderIdErr) return rej('invalid folder_id param'); - // Download file - const data = await download(url); - - // Create file - const driveFile = await create(user, data, name, null, folderId); - - // Serialize - const fileObj = await serialize(driveFile); - - // Response - res(fileObj); + // Create temp file + new Promise((res, rej) => { + tmp.file((e, path) => { + if (e) return rej(e) + res(path) + }) + }) + // Download file + .then((path: string) => new Promise((res, rej) => { + const writable = fs.createWriteStream(path) + request(url) + .on('error', rej) + .on('end', () => { + writable.close() + res(path) + }) + .pipe(writable) + .on('error', rej) + })) + // Calculate hash & content-type + .then((path: string) => new Promise((res, rej) => { + const readable = fs.createReadStream(path) + const hash = crypto.createHash('md5') + readable + .on('error', rej) + .on('end', () => { + hash.end() + res([path, hash.digest('hex')]) + }) + .pipe(hash) + .on('error', rej) + })) + // Create file + .then((rv: string[]) => new Promise((res, rej) => { + const [path, hash] = rv + create(user, { + stream: fs.createReadStream(path), + name, + hash + }, null, folderId) + .then(driveFile => { + res(driveFile) + // crean-up + fs.unlink(path, (e) => { + if (e) log(e.stack) + }) + }) + .catch(rej) + })) + // Serialize + .then(serialize) + .then(res) + .catch(rej) });