Use node-fetch instead of request (#6228)

* requestをnode-fetchになど

* format

* fix error

* t

* Fix test
This commit is contained in:
MeiMei 2020-04-09 23:42:23 +09:00 committed by GitHub
parent bb7edfee04
commit d3c0f3c251
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 140 additions and 218 deletions

View File

@ -72,6 +72,7 @@
"@types/markdown-it": "0.0.9", "@types/markdown-it": "0.0.9",
"@types/mocha": "7.0.2", "@types/mocha": "7.0.2",
"@types/node": "13.11.0", "@types/node": "13.11.0",
"@types/node-fetch": "2.5.5",
"@types/nodemailer": "6.4.0", "@types/nodemailer": "6.4.0",
"@types/nprogress": "0.2.0", "@types/nprogress": "0.2.0",
"@types/oauth": "0.9.1", "@types/oauth": "0.9.1",
@ -84,8 +85,6 @@
"@types/ratelimiter": "2.1.28", "@types/ratelimiter": "2.1.28",
"@types/redis": "2.8.17", "@types/redis": "2.8.17",
"@types/rename": "1.0.1", "@types/rename": "1.0.1",
"@types/request": "2.48.4",
"@types/request-promise-native": "1.0.17",
"@types/request-stats": "3.0.0", "@types/request-stats": "3.0.0",
"@types/rimraf": "2.0.3", "@types/rimraf": "2.0.3",
"@types/seedrandom": "2.4.28", "@types/seedrandom": "2.4.28",
@ -102,7 +101,6 @@
"@types/websocket": "1.0.0", "@types/websocket": "1.0.0",
"@types/ws": "7.2.3", "@types/ws": "7.2.3",
"@typescript-eslint/parser": "2.26.0", "@typescript-eslint/parser": "2.26.0",
"agentkeepalive": "4.1.0",
"animejs": "3.1.0", "animejs": "3.1.0",
"apexcharts": "3.17.1", "apexcharts": "3.17.1",
"autobind-decorator": "2.4.0", "autobind-decorator": "2.4.0",
@ -145,6 +143,7 @@
"gulp-typescript": "5.0.1", "gulp-typescript": "5.0.1",
"hard-source-webpack-plugin": "0.13.1", "hard-source-webpack-plugin": "0.13.1",
"html-minifier": "4.0.0", "html-minifier": "4.0.0",
"http-proxy-agent": "4.0.1",
"http-signature": "1.3.4", "http-signature": "1.3.4",
"https-proxy-agent": "5.0.0", "https-proxy-agent": "5.0.0",
"insert-text-at-cursor": "0.3.0", "insert-text-at-cursor": "0.3.0",
@ -205,8 +204,6 @@
"redis-lock": "0.1.4", "redis-lock": "0.1.4",
"reflect-metadata": "0.1.13", "reflect-metadata": "0.1.13",
"rename": "1.0.4", "rename": "1.0.4",
"request": "2.88.2",
"request-promise-native": "1.0.8",
"request-stats": "3.0.0", "request-stats": "3.0.0",
"require-all": "3.0.0", "require-all": "3.0.0",
"rimraf": "3.0.2", "rimraf": "3.0.2",

View File

@ -1,5 +1,6 @@
import * as fs from 'fs'; import * as fs from 'fs';
import * as request from 'request'; import fetch from 'node-fetch';
import { httpAgent, httpsAgent } from './fetch';
import config from '../config'; import config from '../config';
import * as chalk from 'chalk'; import * as chalk from 'chalk';
import Logger from '../services/logger'; import Logger from '../services/logger';
@ -7,11 +8,35 @@ import Logger from '../services/logger';
export async function downloadUrl(url: string, path: string) { export async function downloadUrl(url: string, path: string) {
const logger = new Logger('download'); const logger = new Logger('download');
await new Promise((res, rej) => { logger.info(`Downloading ${chalk.cyan(url)} ...`);
logger.info(`Downloading ${chalk.cyan(url)} ...`);
const response = await fetch(new URL(url).href, {
headers: {
'User-Agent': config.userAgent
},
timeout: 10 * 1000,
agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
}).then(response => {
if (!response.ok) {
logger.error(`Got ${response.status} (${url})`);
throw response.status;
} else {
return response;
}
});
await new Promise((res, rej) => {
const writable = fs.createWriteStream(path); const writable = fs.createWriteStream(path);
response.body.on('error', (error: any) => {
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
writable.close();
rej(error);
});
writable.on('finish', () => { writable.on('finish', () => {
logger.succ(`Download finished: ${chalk.cyan(url)}`); logger.succ(`Download finished: ${chalk.cyan(url)}`);
res(); res();
@ -25,35 +50,6 @@ export async function downloadUrl(url: string, path: string) {
rej(error); rej(error);
}); });
const req = request({ response.body.pipe(writable);
url: new URL(url).href, // https://github.com/syuilo/misskey/issues/2637
proxy: config.proxy,
timeout: 10 * 1000,
forever: true,
headers: {
'User-Agent': config.userAgent
}
});
req.pipe(writable);
req.on('response', response => {
if (response.statusCode !== 200) {
logger.error(`Got ${response.statusCode} (${url})`);
writable.close();
rej(response.statusCode);
}
});
req.on('error', error => {
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
writable.close();
rej(error);
});
logger.succ(`Downloaded to: ${path}`);
}); });
} }

43
src/misc/fetch.ts Normal file
View File

@ -0,0 +1,43 @@
import * as http from 'http';
import * as https from 'https';
import * as cache from 'lookup-dns-cache';
import fetch, { HeadersInit } from 'node-fetch';
import { HttpProxyAgent } from 'http-proxy-agent';
import { HttpsProxyAgent } from 'https-proxy-agent';
import config from '../config';
export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: HeadersInit) {
const res = await fetch(url, {
headers: Object.assign({
'User-Agent': config.userAgent,
Accept: accept
}, headers || {}),
timeout,
agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
});
if (!res.ok) {
throw {
name: `StatusError`,
statusCode: res.status,
message: `${res.status} ${res.statusText}`,
};
}
return await res.json();
}
export const httpAgent = config.proxy
? new HttpProxyAgent(config.proxy)
: new http.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
});
export const httpsAgent = config.proxy
? new HttpsProxyAgent(config.proxy)
: new https.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup,
});

View File

@ -1,19 +1,12 @@
import * as https from 'https'; import * as https from 'https';
import { sign } from 'http-signature'; import { sign } from 'http-signature';
import * as crypto from 'crypto'; import * as crypto from 'crypto';
import * as cache from 'lookup-dns-cache';
import config from '../../config'; import config from '../../config';
import { ILocalUser } from '../../models/entities/user'; import { ILocalUser } from '../../models/entities/user';
import { UserKeypairs } from '../../models'; import { UserKeypairs } from '../../models';
import { ensure } from '../../prelude/ensure'; import { ensure } from '../../prelude/ensure';
import { HttpsProxyAgent } from 'https-proxy-agent'; import { httpsAgent } from '../../misc/fetch';
const agent = config.proxy
? new HttpsProxyAgent(config.proxy)
: new https.Agent({
lookup: cache.lookup,
});
export default async (user: ILocalUser, url: string, object: any) => { export default async (user: ILocalUser, url: string, object: any) => {
const timeout = 10 * 1000; const timeout = 10 * 1000;
@ -32,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => {
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
const req = https.request({ const req = https.request({
agent, agent: httpsAgent,
protocol, protocol,
hostname, hostname,
port, port,

View File

@ -1,10 +1,8 @@
import * as request from 'request-promise-native'; import { getJson } from '../../misc/fetch';
import { IObject, isCollectionOrOrderedCollection, ICollection, IOrderedCollection } from './type'; import { IObject, isCollectionOrOrderedCollection, ICollection, IOrderedCollection } from './type';
import config from '../../config';
export default class Resolver { export default class Resolver {
private history: Set<string>; private history: Set<string>;
private timeout = 10 * 1000;
constructor() { constructor() {
this.history = new Set(); this.history = new Set();
@ -41,24 +39,7 @@ export default class Resolver {
this.history.add(value); this.history.add(value);
const object = await request({ const object = await getJson(value, 'application/activity+json, application/ld+json');
url: value,
proxy: config.proxy,
timeout: this.timeout,
forever: true,
headers: {
'User-Agent': config.userAgent,
Accept: 'application/activity+json, application/ld+json'
},
json: true
}).catch(e => {
const message = `${e.name}: ${e.message ? e.message.substr(0, 200) : undefined}, url=${value}`;
throw {
name: e.name,
statusCode: e.statusCode,
message,
};
});
if (object == null || ( if (object == null || (
Array.isArray(object['@context']) ? Array.isArray(object['@context']) ?

View File

@ -1,5 +1,4 @@
import config from '../config'; import { getJson } from '../misc/fetch';
import * as request from 'request-promise-native';
import { query as urlQuery } from '../prelude/url'; import { query as urlQuery } from '../prelude/url';
type ILink = { type ILink = {
@ -15,17 +14,7 @@ type IWebFinger = {
export default async function(query: string): Promise<IWebFinger> { export default async function(query: string): Promise<IWebFinger> {
const url = genUrl(query); const url = genUrl(query);
return await request({ return await getJson(url, 'application/jrd+json, application/json');
url,
proxy: config.proxy,
timeout: 10 * 1000,
forever: true,
headers: {
'User-Agent': config.userAgent,
Accept: 'application/jrd+json, application/json'
},
json: true
});
} }
function genUrl(query: string) { function genUrl(query: string) {

View File

@ -1,6 +1,6 @@
import * as Koa from 'koa'; import * as Koa from 'koa';
import * as Router from '@koa/router'; import * as Router from '@koa/router';
import * as request from 'request'; import { getJson } from '../../../misc/fetch';
import { OAuth2 } from 'oauth'; import { OAuth2 } from 'oauth';
import config from '../../../config'; import config from '../../../config';
import { publishMainStream } from '../../../services/stream'; import { publishMainStream } from '../../../services/stream';
@ -174,20 +174,9 @@ router.get('/dc/cb', async ctx => {
} }
})); }));
const { id, username, discriminator } = await new Promise<any>((res, rej) => const { id, username, discriminator } = await getJson('https://discordapp.com/api/users/@me', '*/*', 10 * 1000, {
request({ 'Authorization': `Bearer ${accessToken}`,
url: 'https://discordapp.com/api/users/@me', });
headers: {
'Authorization': `Bearer ${accessToken}`,
'User-Agent': config.userAgent
}
}, (err, response, body) => {
if (err) {
rej(err);
} else {
res(JSON.parse(body));
}
}));
if (!id || !username || !discriminator) { if (!id || !username || !discriminator) {
ctx.throw(400, 'invalid session'); ctx.throw(400, 'invalid session');
@ -256,21 +245,9 @@ router.get('/dc/cb', async ctx => {
} }
})); }));
const { id, username, discriminator } = await new Promise<any>((res, rej) => const { id, username, discriminator } = await getJson('https://discordapp.com/api/users/@me', '*/*', 10 * 1000, {
request({ 'Authorization': `Bearer ${accessToken}`,
url: 'https://discordapp.com/api/users/@me', });
headers: {
'Authorization': `Bearer ${accessToken}`,
'User-Agent': config.userAgent
}
}, (err, response, body) => {
if (err) {
rej(err);
} else {
res(JSON.parse(body));
}
}));
if (!id || !username || !discriminator) { if (!id || !username || !discriminator) {
ctx.throw(400, 'invalid session'); ctx.throw(400, 'invalid session');
return; return;

View File

@ -1,6 +1,6 @@
import * as Koa from 'koa'; import * as Koa from 'koa';
import * as Router from '@koa/router'; import * as Router from '@koa/router';
import * as request from 'request'; import { getJson } from '../../../misc/fetch';
import { OAuth2 } from 'oauth'; import { OAuth2 } from 'oauth';
import config from '../../../config'; import config from '../../../config';
import { publishMainStream } from '../../../services/stream'; import { publishMainStream } from '../../../services/stream';
@ -167,21 +167,9 @@ router.get('/gh/cb', async ctx => {
} }
})); }));
const { login, id } = await new Promise<any>((res, rej) => const { login, id } = await getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
request({ 'Authorization': `bearer ${accessToken}`
url: 'https://api.github.com/user', });
headers: {
'Accept': 'application/vnd.github.v3+json',
'Authorization': `bearer ${accessToken}`,
'User-Agent': config.userAgent
}
}, (err, response, body) => {
if (err)
rej(err);
else
res(JSON.parse(body));
}));
if (!login || !id) { if (!login || !id) {
ctx.throw(400, 'invalid session'); ctx.throw(400, 'invalid session');
return; return;
@ -230,20 +218,9 @@ router.get('/gh/cb', async ctx => {
res({ accessToken }); res({ accessToken });
})); }));
const { login, id } = await new Promise<any>((res, rej) => const { login, id } = await getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
request({ 'Authorization': `bearer ${accessToken}`
url: 'https://api.github.com/user', });
headers: {
'Accept': 'application/vnd.github.v3+json',
'Authorization': `bearer ${accessToken}`,
'User-Agent': config.userAgent
}
}, (err, response, body) => {
if (err)
rej(err);
else
res(JSON.parse(body));
}));
if (!login || !id) { if (!login || !id) {
ctx.throw(400, 'invalid session'); ctx.throw(400, 'invalid session');

View File

@ -1,10 +1,10 @@
import * as Koa from 'koa'; import * as Koa from 'koa';
import * as request from 'request-promise-native';
import summaly from 'summaly'; import summaly from 'summaly';
import { fetchMeta } from '../../misc/fetch-meta'; import { fetchMeta } from '../../misc/fetch-meta';
import Logger from '../../services/logger'; import Logger from '../../services/logger';
import config from '../../config'; import config from '../../config';
import { query } from '../../prelude/url'; import { query } from '../../prelude/url';
import { getJson } from '../../misc/fetch';
const logger = new Logger('url-preview'); const logger = new Logger('url-preview');
@ -16,15 +16,10 @@ module.exports = async (ctx: Koa.Context) => {
: `Getting preview of ${ctx.query.url}@${ctx.query.lang} ...`); : `Getting preview of ${ctx.query.url}@${ctx.query.lang} ...`);
try { try {
const summary = meta.summalyProxy ? await request.get({ const summary = meta.summalyProxy ? await getJson(`${meta.summalyProxy}?${query({
url: meta.summalyProxy, url: ctx.query.url,
qs: { lang: ctx.query.lang || 'ja-JP'
url: ctx.query.url, })}`) : await summaly(ctx.query.url, {
lang: ctx.query.lang || 'ja-JP'
},
forever: true,
json: true
}) : await summaly(ctx.query.url, {
followRedirects: false, followRedirects: false,
lang: ctx.query.lang || 'ja-JP' lang: ctx.query.lang || 'ja-JP'
}); });

View File

@ -1,32 +1,17 @@
import * as S3 from 'aws-sdk/clients/s3'; import * as S3 from 'aws-sdk/clients/s3';
import config from '../../config';
import { Meta } from '../../models/entities/meta'; import { Meta } from '../../models/entities/meta';
import { HttpsProxyAgent } from 'https-proxy-agent'; import { httpsAgent, httpAgent } from '../../misc/fetch';
import * as agentkeepalive from 'agentkeepalive';
const httpsAgent = config.proxy
? new HttpsProxyAgent(config.proxy)
: new agentkeepalive.HttpsAgent({
freeSocketTimeout: 30 * 1000
});
export function getS3(meta: Meta) { export function getS3(meta: Meta) {
const conf = { return new S3({
endpoint: meta.objectStorageEndpoint || undefined, endpoint: meta.objectStorageEndpoint || undefined,
accessKeyId: meta.objectStorageAccessKey, accessKeyId: meta.objectStorageAccessKey!,
secretAccessKey: meta.objectStorageSecretKey, secretAccessKey: meta.objectStorageSecretKey!,
region: meta.objectStorageRegion || undefined, region: meta.objectStorageRegion || undefined,
sslEnabled: meta.objectStorageUseSSL, sslEnabled: meta.objectStorageUseSSL,
s3ForcePathStyle: !!meta.objectStorageEndpoint, s3ForcePathStyle: !!meta.objectStorageEndpoint,
httpOptions: { httpOptions: {
agent: meta.objectStorageUseSSL ? httpsAgent : httpAgent
} }
} as S3.ClientConfiguration; });
if (meta.objectStorageUseSSL) {
conf.httpOptions!.agent = httpsAgent;
}
const s3 = new S3(conf);
return s3;
} }

View File

@ -1,7 +1,6 @@
import * as request from 'request-promise-native'; import { getJson } from '../misc/fetch';
import { Instance } from '../models/entities/instance'; import { Instance } from '../models/entities/instance';
import { Instances } from '../models'; import { Instances } from '../models';
import config from '../config';
import { getNodeinfoLock } from '../misc/app-lock'; import { getNodeinfoLock } from '../misc/app-lock';
import Logger from '../services/logger'; import Logger from '../services/logger';
@ -20,23 +19,14 @@ export async function fetchNodeinfo(instance: Instance) {
logger.info(`Fetching nodeinfo of ${instance.host} ...`); logger.info(`Fetching nodeinfo of ${instance.host} ...`);
try { try {
const wellknown = await request({ const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
url: 'https://' + instance.host + '/.well-known/nodeinfo', .catch(e => {
proxy: config.proxy, if (e.statusCode === 404) {
timeout: 1000 * 10, throw 'No nodeinfo provided';
forever: true, } else {
headers: { throw e.statusCode || e.message;
'User-Agent': config.userAgent, }
Accept: 'application/json, */*' });
},
json: true
}).catch(e => {
if (e.statusCode === 404) {
throw 'No nodeinfo provided';
} else {
throw e.statusCode || e.message;
}
});
if (wellknown.links == null || !Array.isArray(wellknown.links)) { if (wellknown.links == null || !Array.isArray(wellknown.links)) {
throw 'No wellknown links'; throw 'No wellknown links';
@ -53,19 +43,10 @@ export async function fetchNodeinfo(instance: Instance) {
throw 'No nodeinfo link provided'; throw 'No nodeinfo link provided';
} }
const info = await request({ const info = await getJson(link.href)
url: link.href, .catch(e => {
proxy: config.proxy, throw e.statusCode || e.message;
timeout: 1000 * 10, });
forever: true,
headers: {
'User-Agent': config.userAgent,
Accept: 'application/json, */*'
},
json: true
}).catch(e => {
throw e.statusCode || e.message;
});
await Instances.update(instance.id, { await Instances.update(instance.id, {
infoUpdatedAt: new Date(), infoUpdatedAt: new Date(),

View File

@ -1,7 +1,7 @@
import * as fs from 'fs'; import * as fs from 'fs';
import * as WebSocket from 'ws'; import * as WebSocket from 'ws';
const fetch = require('node-fetch'); import fetch from 'node-fetch';
import * as req from 'request'; const FormData = require('form-data');
import * as childProcess from 'child_process'; import * as childProcess from 'child_process';
export const async = (fn: Function) => (done: Function) => { export const async = (fn: Function) => (done: Function) => {
@ -20,6 +20,9 @@ export const request = async (endpoint: string, params: any, me?: any): Promise<
try { try {
const res = await fetch('http://localhost:8080/api' + endpoint, { const res = await fetch('http://localhost:8080/api' + endpoint, {
method: 'POST', method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(Object.assign(auth, params)) body: JSON.stringify(Object.assign(auth, params))
}); });
@ -64,18 +67,23 @@ export const react = async (user: any, note: any, reaction: string): Promise<any
}, user); }, user);
}; };
export const uploadFile = (user: any, path?: string): Promise<any> => new Promise((ok, rej) => { export const uploadFile = (user: any, path?: string): Promise<any> => {
req.post({ const formData = new FormData();
url: 'http://localhost:8080/api/drive/files/create', formData.append('i', user.token);
formData: { formData.append('file', fs.createReadStream(path || __dirname + '/resources/Lenna.png'));
i: user.token,
file: fs.createReadStream(path || __dirname + '/resources/Lenna.png') return fetch('http://localhost:8080/api/drive/files/create', {
}, method: 'post',
json: true body: formData,
}, (err, httpResponse, body) => { timeout: 30 * 1000,
ok(body); }).then(res => {
}); if (!res.ok) {
}); throw `${res.status} ${res.statusText}`;
} else {
return res.json();
}
});
};
export function connectStream(user: any, channel: string, listener: (message: Record<string, any>) => any, params?: any): Promise<WebSocket> { export function connectStream(user: any, channel: string, listener: (message: Record<string, any>) => any, params?: any): Promise<WebSocket> {
return new Promise((res, rej) => { return new Promise((res, rej) => {