From 324d1a932d313f524181df89603d95cf254d9629 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sat, 15 Jul 2023 23:02:00 -0400 Subject: [PATCH 1/7] handle import mastodon package --- .../backend/src/misc/process-masto-notes.ts | 58 +++++++++++++++++++ .../queue/processors/db/import-masto-post.ts | 33 ++++++----- .../src/queue/processors/db/import-posts.ts | 17 ++++++ 3 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 packages/backend/src/misc/process-masto-notes.ts diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts new file mode 100644 index 000000000..1310748d2 --- /dev/null +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -0,0 +1,58 @@ +import * as fs from "node:fs"; +import Logger from "@/services/logger.js"; +import { createTemp, createTempDir } from "./create-temp.js"; +import { downloadUrl } from "./download-url.js"; +import { addFile } from "@/services/drive/add-file.js"; +import { exec } from "node:child_process"; +import { Users } from "@/models/index.js"; + +const logger = new Logger("download-text-file"); + +export async function processMastoNotes( + url: string, + uid: string, +): Promise { + // Create temp file + const [path, cleanup] = await createTemp(); + + const [unzipPath, unzipCleanup] = await createTempDir(); + + logger.info(`Temp file is ${path}`); + + try { + // write content at URL to temp file + await downloadUrl(url, path); + return await processMastoFile(path, unzipPath, uid); + } finally { + cleanup(); + unzipCleanup(); + } +} + +function processMastoFile(fn: string, dir: string, uid: string) { + return new Promise(async (resolve, reject) => { + const user = await Users.findOneBy({ id: uid }); + exec( + `tar -xf ${fn} -C ${dir}`, + async (error: any, stdout: string, stderr: string) => { + if (error) { + reject(error); + } + const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); + for (const note of outbox.orderedItems) { + for (const attachment of note.object.attachment) { + const url = attachment.url.replace("..", ""); + try { + const fpath = `${dir}${url}`; + const driveFile = await addFile({ user: user, path: fpath }); + attachment.driveFile = driveFile; + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } + } + resolve(outbox); + }, + ); + }); +} diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts index 05166b085..1d18008a0 100644 --- a/packages/backend/src/queue/processors/db/import-masto-post.ts +++ b/packages/backend/src/queue/processors/db/import-masto-post.ts @@ -45,19 +45,26 @@ export async function importMastoPost( throw e; } job.progress(80); - const urls = post.object.attachment - .map((x: any) => x.url) - .filter((x: String) => x.startsWith("http")); - const files: DriveFile[] = []; - for (const url of urls) { - try { - const file = await uploadFromUrl({ - url: url, - user: user, - }); - files.push(file); - } catch (e) { - logger.error(`Skipped adding file to drive: ${url}`); + + let files: DriveFile[] = (post.object.attachment || []) + .map((x: any) => x?.driveFile) + .filter((x: any) => x); + + if (files.length == 0) { + const urls = post.object.attachment + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + files = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } } } diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index f92a5f710..54061f8bf 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -1,4 +1,5 @@ import { downloadTextFile } from "@/misc/download-text-file.js"; +import { processMastoNotes } from "@/misc/process-masto-notes.js"; import { Users, DriveFiles } from "@/models/index.js"; import type { DbUserImportPostsJobData } from "@/queue/types.js"; import { queueLogger } from "../../logger.js"; @@ -30,6 +31,22 @@ export async function importPosts( return; } + if (file.name.endsWith("tar.gz")) { + try { + logger.info("Parsing animal style posts in package"); + const outbox = await processMastoNotes(file.url, job.data.user.id); + for (const post of outbox.orderedItems) { + createImportMastoPostJob(job.data.user, post, job.data.signatureCheck); + } + } catch (e) { + // handle error + logger.warn(`Error reading: ${e}`); + } + logger.succ("Imported"); + done(); + return; + } + const json = await downloadTextFile(file.url); try { From 809d4180185a12d9eff48562e81caea864b4f5b3 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 01:41:57 -0400 Subject: [PATCH 2/7] use gunzip-maybe tar-stream to replace exec --- packages/backend/package.json | 2 + .../backend/src/misc/process-masto-notes.ts | 87 ++++++++++++++----- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index fe8c078a0..6e9f82edd 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -65,6 +65,7 @@ "file-type": "17.1.6", "fluent-ffmpeg": "2.1.2", "got": "12.5.3", + "gunzip-maybe": "^1.4.2", "hpagent": "0.1.2", "ioredis": "5.3.2", "ip-cidr": "3.1.0", @@ -125,6 +126,7 @@ "summaly": "2.7.0", "syslog-pro": "1.0.0", "systeminformation": "5.17.17", + "tar-stream": "^3.1.6", "tesseract.js": "^3.0.3", "tinycolor2": "1.5.2", "tmp": "0.2.1", diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts index 1310748d2..2fe7c7a58 100644 --- a/packages/backend/src/misc/process-masto-notes.ts +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -3,10 +3,11 @@ import Logger from "@/services/logger.js"; import { createTemp, createTempDir } from "./create-temp.js"; import { downloadUrl } from "./download-url.js"; import { addFile } from "@/services/drive/add-file.js"; -import { exec } from "node:child_process"; import { Users } from "@/models/index.js"; +import * as tar from 'tar-stream'; +import gunzip from "gunzip-maybe"; -const logger = new Logger("download-text-file"); +const logger = new Logger("process-masto-notes"); export async function processMastoNotes( url: string, @@ -32,27 +33,69 @@ export async function processMastoNotes( function processMastoFile(fn: string, dir: string, uid: string) { return new Promise(async (resolve, reject) => { const user = await Users.findOneBy({ id: uid }); - exec( - `tar -xf ${fn} -C ${dir}`, - async (error: any, stdout: string, stderr: string) => { - if (error) { - reject(error); - } - const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); - for (const note of outbox.orderedItems) { - for (const attachment of note.object.attachment) { - const url = attachment.url.replace("..", ""); - try { - const fpath = `${dir}${url}`; - const driveFile = await addFile({ user: user, path: fpath }); - attachment.driveFile = driveFile; - } catch (e) { - logger.error(`Skipped adding file to drive: ${url}`); - } + try{ + logger.info(`Start unzip ${fn}`); + await unzipTarGz(fn, dir); + logger.info(`Unzip to ${dir}`); + const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); + for (const note of outbox.orderedItems) { + for (const attachment of note.object.attachment) { + const url = attachment.url.replace("..", ""); + try { + const fpath = `${dir}${url}`; + const driveFile = await addFile({ user: user, path: fpath }); + attachment.driveFile = driveFile; + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); } } - resolve(outbox); - }, - ); + } + resolve(outbox); + }catch(e){ + logger.error(`Error on extract masto note package: ${fn}`); + reject(e); + } }); } + +function createFileDir(fn: string){ + if(!fs.existsSync(fn)){ + fs.mkdirSync(fn, {recursive: true}); + fs.rmdirSync(fn); + } +} + +function unzipTarGz(fn: string, dir: string){ + return new Promise(async (resolve, reject) => { + const onErr = (err: any) => { + logger.error(`pipe broken: ${err}`); + reject(); + } + try{ + const extract = tar.extract().on('error', onErr); + dir = dir.endsWith("/") ? dir : dir + "/"; + const ls: string[] = []; + extract.on('entry', function (header: any, stream: any, next: any) { + try{ + ls.push(dir + header.name); + createFileDir(dir + header.name); + stream.on('error', onErr).pipe(fs.createWriteStream(dir + header.name)).on('error', onErr); + next(); + }catch(e){ + logger.error(`create dir error:${e}`); + reject(); + } + }); + + extract.on('finish', function () { + resolve(ls); + }); + + fs.createReadStream(fn).on('error', onErr).pipe(gunzip()).on('error', onErr).pipe(extract).on('error', onErr); + + }catch(e){ + logger.error(`unzipTarGz error: ${e}`); + reject(); + } + }); +} \ No newline at end of file From 0c9ab9fdfa5ea1e0be71b6abb680e5c7ef97c116 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 01:47:06 -0400 Subject: [PATCH 3/7] fix format --- .../backend/src/misc/process-masto-notes.ts | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts index 2fe7c7a58..f51764ae8 100644 --- a/packages/backend/src/misc/process-masto-notes.ts +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -4,7 +4,7 @@ import { createTemp, createTempDir } from "./create-temp.js"; import { downloadUrl } from "./download-url.js"; import { addFile } from "@/services/drive/add-file.js"; import { Users } from "@/models/index.js"; -import * as tar from 'tar-stream'; +import * as tar from "tar-stream"; import gunzip from "gunzip-maybe"; const logger = new Logger("process-masto-notes"); @@ -33,7 +33,7 @@ export async function processMastoNotes( function processMastoFile(fn: string, dir: string, uid: string) { return new Promise(async (resolve, reject) => { const user = await Users.findOneBy({ id: uid }); - try{ + try { logger.info(`Start unzip ${fn}`); await unzipTarGz(fn, dir); logger.info(`Unzip to ${dir}`); @@ -51,51 +51,58 @@ function processMastoFile(fn: string, dir: string, uid: string) { } } resolve(outbox); - }catch(e){ + } catch (e) { logger.error(`Error on extract masto note package: ${fn}`); reject(e); } }); } -function createFileDir(fn: string){ - if(!fs.existsSync(fn)){ - fs.mkdirSync(fn, {recursive: true}); +function createFileDir(fn: string) { + if (!fs.existsSync(fn)) { + fs.mkdirSync(fn, { recursive: true }); fs.rmdirSync(fn); } } -function unzipTarGz(fn: string, dir: string){ +function unzipTarGz(fn: string, dir: string) { return new Promise(async (resolve, reject) => { const onErr = (err: any) => { logger.error(`pipe broken: ${err}`); reject(); - } - try{ - const extract = tar.extract().on('error', onErr); + }; + try { + const extract = tar.extract().on("error", onErr); dir = dir.endsWith("/") ? dir : dir + "/"; const ls: string[] = []; - extract.on('entry', function (header: any, stream: any, next: any) { - try{ + extract.on("entry", function (header: any, stream: any, next: any) { + try { ls.push(dir + header.name); createFileDir(dir + header.name); - stream.on('error', onErr).pipe(fs.createWriteStream(dir + header.name)).on('error', onErr); + stream + .on("error", onErr) + .pipe(fs.createWriteStream(dir + header.name)) + .on("error", onErr); next(); - }catch(e){ + } catch (e) { logger.error(`create dir error:${e}`); reject(); } }); - extract.on('finish', function () { - resolve(ls); + extract.on("finish", function () { + resolve(ls); }); - - fs.createReadStream(fn).on('error', onErr).pipe(gunzip()).on('error', onErr).pipe(extract).on('error', onErr); - - }catch(e){ + + fs.createReadStream(fn) + .on("error", onErr) + .pipe(gunzip()) + .on("error", onErr) + .pipe(extract) + .on("error", onErr); + } catch (e) { logger.error(`unzipTarGz error: ${e}`); reject(); } }); -} \ No newline at end of file +} From 35f09d00674ea742cfa833e6dd0ba81b4726ab76 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 03:24:37 -0400 Subject: [PATCH 4/7] change wording --- packages/backend/src/queue/processors/db/import-posts.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index 54061f8bf..0c5280ae1 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -33,16 +33,16 @@ export async function importPosts( if (file.name.endsWith("tar.gz")) { try { - logger.info("Parsing animal style posts in package"); + logger.info("Reading Mastodon archive"); const outbox = await processMastoNotes(file.url, job.data.user.id); for (const post of outbox.orderedItems) { createImportMastoPostJob(job.data.user, post, job.data.signatureCheck); } } catch (e) { // handle error - logger.warn(`Error reading: ${e}`); + logger.warn(`Failed reading Mastodon archive: ${e}`); } - logger.succ("Imported"); + logger.succ("Mastodon archive imported"); done(); return; } From ffb7fb5309946e42b8e9af9fa34ff7d60c9b1cd5 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 04:07:55 -0400 Subject: [PATCH 5/7] Update pnpm-lock.yaml --- pnpm-lock.yaml | 165 +++++++++++++++++++++++++------------------------ 1 file changed, 85 insertions(+), 80 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index adead9c6f..4fbb6870d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -201,6 +201,9 @@ importers: got: specifier: 12.5.3 version: 12.5.3 + gunzip-maybe: + specifier: ^1.4.2 + version: 1.4.2 hpagent: specifier: 0.1.2 version: 0.1.2 @@ -381,6 +384,9 @@ importers: systeminformation: specifier: 5.17.17 version: 5.17.17 + tar-stream: + specifier: ^3.1.6 + version: 3.1.6 tesseract.js: specifier: ^3.0.3 version: 3.0.3(eslint@8.44.0) @@ -595,7 +601,7 @@ importers: version: 5.1.6 webpack: specifier: ^5.88.1 - version: 5.88.1(@swc/core@1.3.68) + version: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) ws: specifier: 8.13.0 version: 8.13.0 @@ -780,7 +786,7 @@ importers: version: 2.30.0 emojilib: specifier: github:thatonecalculator/emojilib - version: github.com/thatonecalculator/emojilib/1d6adc1af8105b4937d2f0e7479acf5bd565bd9b + version: github.com/thatonecalculator/emojilib/d3c8c6a77d4362b3b3180099f1d2eac344ce245c escape-regexp: specifier: 0.0.1 version: 0.0.1 @@ -5138,7 +5144,7 @@ packages: /axios@0.24.0: resolution: {integrity: sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==} dependencies: - follow-redirects: 1.15.2 + follow-redirects: 1.15.2(debug@4.3.4) transitivePeerDependencies: - debug dev: false @@ -5164,13 +5170,17 @@ packages: /axios@1.4.0: resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==} dependencies: - follow-redirects: 1.15.2 + follow-redirects: 1.15.2(debug@4.3.4) form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug dev: false + /b4a@1.6.4: + resolution: {integrity: sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==} + dev: false + /babel-eslint@10.1.0(eslint@8.44.0): resolution: {integrity: sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==} engines: {node: '>=6'} @@ -5479,6 +5489,12 @@ packages: resolution: {integrity: sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==} dev: true + /browserify-zlib@0.1.4: + resolution: {integrity: sha512-19OEpq7vWgsH6WkvkBJQDFvJS1uPcbFOQ4v9CU839dO+ZZXUZO6XpE6hNCqvlIIj+4fZvRiJ6DsAQ382GwiyTQ==} + dependencies: + pako: 0.2.9 + dev: false + /browserslist@1.7.7: resolution: {integrity: sha512-qHJblDE2bXVRYzuDetv/wAeHOJyO97+9wxC1cdCtyzgNuSozOyRCiiLaCR1f71AN66lQdVVBipWm63V+a7bPOw==} deprecated: Browserslist 2 could fail on reading Browserslist >3.0 config used in other tools. @@ -6843,17 +6859,6 @@ packages: dependencies: ms: 2.0.0 - /debug@3.2.7: - resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - dependencies: - ms: 2.1.3 - dev: false - /debug@3.2.7(supports-color@8.1.1): resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -6864,7 +6869,6 @@ packages: dependencies: ms: 2.1.3 supports-color: 8.1.1 - dev: true /debug@4.3.3: resolution: {integrity: sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==} @@ -7204,7 +7208,6 @@ packages: inherits: 2.0.4 readable-stream: 2.3.8 stream-shift: 1.0.1 - dev: true /each-props@1.3.2: resolution: {integrity: sha512-vV0Hem3zAGkJAyU7JSjixeU66rwdynTAa1vofCrSA5fEln+m67Az9CcnkVD776/fsN/UjIWmBDoNRS6t6G9RfA==} @@ -8106,6 +8109,10 @@ packages: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} dev: true + /fast-fifo@1.3.0: + resolution: {integrity: sha512-IgfweLvEpwyA4WgiQe9Nx6VV2QkML2NkvZnk1oKnIzXgXdWxuhF7zw4DvLTPZJn6PIUneiAXPF24QmoEqHTjyw==} + dev: false + /fast-glob@3.3.0: resolution: {integrity: sha512-ChDuvbOypPuNjO8yIDf36x7BlZX1smcUMTTcyoIjycexOxd6DFsKsg21qVBzEmr3G7fUKIRy2/psii+CIUt7FA==} engines: {node: '>=8.6.0'} @@ -8377,16 +8384,6 @@ packages: tabbable: 6.2.0 dev: true - /follow-redirects@1.15.2: - resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - dev: false - /follow-redirects@1.15.2(debug@4.3.4): resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} engines: {node: '>=4.0'} @@ -9057,6 +9054,18 @@ packages: glogg: 1.0.2 dev: true + /gunzip-maybe@1.4.2: + resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} + hasBin: true + dependencies: + browserify-zlib: 0.1.4 + is-deflate: 1.0.0 + is-gzip: 1.0.0 + peek-stream: 1.1.3 + pumpify: 1.5.1 + through2: 2.0.5 + dev: false + /hammerjs@2.0.8: resolution: {integrity: sha512-tSQXBXS/MWQOn/RKckawJ61vvsDpCom87JgxiYdGwHdOa0ht0vzUWDlfioofFCRU0L+6NGDt6XzbgoJvZkMeRQ==} engines: {node: '>=0.8.0'} @@ -9355,7 +9364,7 @@ packages: engines: {node: '>= 4.5.0'} dependencies: agent-base: 4.3.0 - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color dev: false @@ -9716,6 +9725,10 @@ packages: has-tostringtag: 1.0.0 dev: true + /is-deflate@1.0.0: + resolution: {integrity: sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==} + dev: false + /is-descriptor@0.1.6: resolution: {integrity: sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==} engines: {node: '>=0.10.0'} @@ -9804,6 +9817,11 @@ packages: dependencies: is-extglob: 2.1.1 + /is-gzip@1.0.0: + resolution: {integrity: sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==} + engines: {node: '>=0.10.0'} + dev: false + /is-installed-globally@0.4.0: resolution: {integrity: sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==} engines: {node: '>=10'} @@ -11280,7 +11298,7 @@ packages: json5: 2.2.3 loader-utils: 2.0.4 schema-utils: 3.3.0 - webpack: 5.88.1(@swc/core@1.3.68) + webpack: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) dev: true /json5@1.0.2: @@ -11535,7 +11553,7 @@ packages: resolution: {integrity: sha512-UqyYyH5YEXaJrf9S8E23GoJFQZXkBVJ9zYYMPGz919MSX1KuvAcycIuS0ci150HCoPf4XQVhQ84Qf8xRPWxFaQ==} engines: {node: '>= 7.6.0'} dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) koa-send: 5.0.1 transitivePeerDependencies: - supports-color @@ -12579,7 +12597,7 @@ packages: engines: {node: '>= 4.4.x'} hasBin: true dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@8.1.1) iconv-lite: 0.4.24 sax: 1.2.4 transitivePeerDependencies: @@ -13135,6 +13153,10 @@ packages: resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} dev: false + /pako@0.2.9: + resolution: {integrity: sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==} + dev: false + /parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -13319,6 +13341,14 @@ packages: resolution: {integrity: sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==} engines: {node: '>=14.16'} + /peek-stream@1.1.3: + resolution: {integrity: sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==} + dependencies: + buffer-from: 1.1.2 + duplexify: 3.7.1 + through2: 2.0.5 + dev: false + /pend@1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} dev: true @@ -14021,7 +14051,6 @@ packages: dependencies: end-of-stream: 1.4.4 once: 1.4.0 - dev: true /pump@3.0.0: resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} @@ -14035,7 +14064,6 @@ packages: duplexify: 3.7.1 inherits: 2.0.4 pump: 2.0.1 - dev: true /punycode@1.3.2: resolution: {integrity: sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==} @@ -14128,6 +14156,10 @@ packages: /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + /queue-tick@1.0.1: + resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} + dev: false + /quick-lru@4.0.1: resolution: {integrity: sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==} engines: {node: '>=8'} @@ -15278,13 +15310,19 @@ packages: /stream-shift@1.0.1: resolution: {integrity: sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==} - dev: true /streamsearch@1.1.0: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} dev: false + /streamx@2.15.0: + resolution: {integrity: sha512-HcxY6ncGjjklGs1xsP1aR71INYcsXFJet5CU1CHqihQ2J5nOsbd4OjgjHO42w/4QNv9gZb3BueV+Vxok5pLEXg==} + dependencies: + fast-fifo: 1.3.0 + queue-tick: 1.0.1 + dev: false + /strict-event-emitter-types@2.0.0: resolution: {integrity: sha512-Nk/brWYpD85WlOgzw5h173aci0Teyv8YdIAEtV+N88nDB0dLlazZyJMIsN6eo1/AR61l+p6CJTG1JIyFaoNEEA==} dev: true @@ -15558,7 +15596,7 @@ packages: webpack: '>=2' dependencies: '@swc/core': 1.3.68 - webpack: 5.88.1(@swc/core@1.3.68) + webpack: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) dev: true /swiper@10.0.4: @@ -15616,6 +15654,14 @@ packages: readable-stream: 3.6.2 dev: false + /tar-stream@3.1.6: + resolution: {integrity: sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==} + dependencies: + b4a: 1.6.4 + fast-fifo: 1.3.0 + streamx: 2.15.0 + dev: false + /tar@4.4.19: resolution: {integrity: sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA==} engines: {node: '>=4.5'} @@ -15677,7 +15723,7 @@ packages: schema-utils: 3.3.0 serialize-javascript: 6.0.1 terser: 5.19.0 - webpack: 5.88.1(@swc/core@1.3.68) + webpack: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) dev: true /terser@5.19.0: @@ -15779,7 +15825,6 @@ packages: dependencies: readable-stream: 2.3.8 xtend: 4.0.2 - dev: true /through2@4.0.2: resolution: {integrity: sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==} @@ -16016,7 +16061,7 @@ packages: micromatch: 4.0.5 semver: 7.5.4 typescript: 5.1.6 - webpack: 5.88.1(@swc/core@1.3.68) + webpack: 5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3) dev: true /ts-node@10.4.0(@swc/core@1.3.68)(@types/node@20.3.1)(typescript@5.1.3): @@ -16933,46 +16978,6 @@ packages: engines: {node: '>=10.13.0'} dev: true - /webpack@5.88.1(@swc/core@1.3.68): - resolution: {integrity: sha512-FROX3TxQnC/ox4N+3xQoWZzvGXSuscxR32rbzjpXgEzWudJFEJBpdlkkob2ylrv5yzzufD1zph1OoFsLtm6stQ==} - engines: {node: '>=10.13.0'} - hasBin: true - peerDependencies: - webpack-cli: '*' - peerDependenciesMeta: - webpack-cli: - optional: true - dependencies: - '@types/eslint-scope': 3.7.4 - '@types/estree': 1.0.1 - '@webassemblyjs/ast': 1.11.6 - '@webassemblyjs/wasm-edit': 1.11.6 - '@webassemblyjs/wasm-parser': 1.11.6 - acorn: 8.10.0 - acorn-import-assertions: 1.9.0(acorn@8.10.0) - browserslist: 4.21.9 - chrome-trace-event: 1.0.3 - enhanced-resolve: 5.15.0 - es-module-lexer: 1.3.0 - eslint-scope: 5.1.1 - events: 3.3.0 - glob-to-regexp: 0.4.1 - graceful-fs: 4.2.11 - json-parse-even-better-errors: 2.3.1 - loader-runner: 4.3.0 - mime-types: 2.1.35 - neo-async: 2.6.2 - schema-utils: 3.3.0 - tapable: 2.2.1 - terser-webpack-plugin: 5.3.9(@swc/core@1.3.68)(webpack@5.88.1) - watchpack: 2.4.0 - webpack-sources: 3.2.3 - transitivePeerDependencies: - - '@swc/core' - - esbuild - - uglify-js - dev: true - /webpack@5.88.1(@swc/core@1.3.68)(webpack-cli@5.1.3): resolution: {integrity: sha512-FROX3TxQnC/ox4N+3xQoWZzvGXSuscxR32rbzjpXgEzWudJFEJBpdlkkob2ylrv5yzzufD1zph1OoFsLtm6stQ==} engines: {node: '>=10.13.0'} @@ -17486,8 +17491,8 @@ packages: url-polyfill: 1.1.12 dev: true - github.com/thatonecalculator/emojilib/1d6adc1af8105b4937d2f0e7479acf5bd565bd9b: - resolution: {tarball: https://codeload.github.com/thatonecalculator/emojilib/tar.gz/1d6adc1af8105b4937d2f0e7479acf5bd565bd9b} + github.com/thatonecalculator/emojilib/d3c8c6a77d4362b3b3180099f1d2eac344ce245c: + resolution: {tarball: https://codeload.github.com/thatonecalculator/emojilib/tar.gz/d3c8c6a77d4362b3b3180099f1d2eac344ce245c} name: emojilib version: 3.0.10 dev: true From 4bb6d7105a8b0509d6579af98bdb08fd7f04370c Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 06:24:37 -0400 Subject: [PATCH 6/7] support new zip file by mastodon --- packages/backend/package.json | 1 + .../backend/src/misc/process-masto-notes.ts | 42 ++++- .../src/queue/processors/db/import-posts.ts | 8 +- pnpm-lock.yaml | 161 +++++++++++++++++- 4 files changed, 196 insertions(+), 16 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 6e9f82edd..d05583845 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -59,6 +59,7 @@ "color-convert": "2.0.1", "content-disposition": "0.5.4", "date-fns": "2.30.0", + "decompress": "^4.2.1", "deep-email-validator": "0.1.21", "escape-regexp": "0.0.1", "feed": "4.2.2", diff --git a/packages/backend/src/misc/process-masto-notes.ts b/packages/backend/src/misc/process-masto-notes.ts index f51764ae8..b8c1c426d 100644 --- a/packages/backend/src/misc/process-masto-notes.ts +++ b/packages/backend/src/misc/process-masto-notes.ts @@ -6,10 +6,13 @@ import { addFile } from "@/services/drive/add-file.js"; import { Users } from "@/models/index.js"; import * as tar from "tar-stream"; import gunzip from "gunzip-maybe"; +import decompress from "decompress"; +import * as Path from "node:path"; const logger = new Logger("process-masto-notes"); export async function processMastoNotes( + fn: string, url: string, uid: string, ): Promise { @@ -23,26 +26,39 @@ export async function processMastoNotes( try { // write content at URL to temp file await downloadUrl(url, path); - return await processMastoFile(path, unzipPath, uid); + return await processMastoFile(fn, path, unzipPath, uid); } finally { cleanup(); - unzipCleanup(); + //unzipCleanup(); } } -function processMastoFile(fn: string, dir: string, uid: string) { +function processMastoFile(fn: string, path: string, dir: string, uid: string) { return new Promise(async (resolve, reject) => { const user = await Users.findOneBy({ id: uid }); try { - logger.info(`Start unzip ${fn}`); - await unzipTarGz(fn, dir); + logger.info(`Start unzip ${path}`); + fn.endsWith("tar.gz") + ? await unzipTarGz(path, dir) + : await unzipZip(path, dir); logger.info(`Unzip to ${dir}`); const outbox = JSON.parse(fs.readFileSync(`${dir}/outbox.json`)); for (const note of outbox.orderedItems) { for (const attachment of note.object.attachment) { - const url = attachment.url.replace("..", ""); + const url = attachment.url.replaceAll("..", ""); + if (url.indexOf('\0') !== -1) { + logger.error(`Found Poison Null Bytes Attack: ${url}`); + reject(); + return; + } try { - const fpath = `${dir}${url}`; + const fpath = Path.resolve(`${dir}${url}`); + if (!fpath.startsWith(dir)) { + logger.error(`Found Path Attack: ${url}`); + reject(); + return; + } + logger.info(fpath); const driveFile = await addFile({ user: user, path: fpath }); attachment.driveFile = driveFile; } catch (e) { @@ -65,6 +81,18 @@ function createFileDir(fn: string) { } } +function unzipZip(fn: string, dir: string) { + return new Promise(async (resolve, reject) => { + try { + decompress(fn, dir).then((files: any) => { + resolve(files); + }); + } catch (e) { + reject(); + } + }); +} + function unzipTarGz(fn: string, dir: string) { return new Promise(async (resolve, reject) => { const onErr = (err: any) => { diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index 0c5280ae1..9bde7479e 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -31,10 +31,14 @@ export async function importPosts( return; } - if (file.name.endsWith("tar.gz")) { + if (file.name.endsWith("tar.gz") || file.name.endsWith("zip")) { try { logger.info("Reading Mastodon archive"); - const outbox = await processMastoNotes(file.url, job.data.user.id); + const outbox = await processMastoNotes( + file.name, + file.url, + job.data.user.id, + ); for (const post of outbox.orderedItems) { createImportMastoPostJob(job.data.user, post, job.data.signatureCheck); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4fbb6870d..8ad191710 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -183,6 +183,9 @@ importers: date-fns: specifier: 2.30.0 version: 2.30.0 + decompress: + specifier: ^4.2.1 + version: 4.2.1 deep-email-validator: specifier: 0.1.21 version: 0.1.21 @@ -5402,6 +5405,13 @@ packages: engines: {node: '>=0.8'} dev: true + /bl@1.2.3: + resolution: {integrity: sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==} + dependencies: + readable-stream: 2.3.8 + safe-buffer: 5.2.1 + dev: false + /bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} dependencies: @@ -5527,6 +5537,17 @@ packages: node-int64: 0.4.0 dev: true + /buffer-alloc-unsafe@1.1.0: + resolution: {integrity: sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==} + dev: false + + /buffer-alloc@1.2.0: + resolution: {integrity: sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==} + dependencies: + buffer-alloc-unsafe: 1.1.0 + buffer-fill: 1.0.0 + dev: false + /buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} @@ -5539,6 +5560,10 @@ packages: engines: {node: '>=0.4'} dev: true + /buffer-fill@1.0.0: + resolution: {integrity: sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==} + dev: false + /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -6930,6 +6955,59 @@ packages: dependencies: mimic-response: 3.1.0 + /decompress-tar@4.1.1: + resolution: {integrity: sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==} + engines: {node: '>=4'} + dependencies: + file-type: 5.2.0 + is-stream: 1.1.0 + tar-stream: 1.6.2 + dev: false + + /decompress-tarbz2@4.1.1: + resolution: {integrity: sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==} + engines: {node: '>=4'} + dependencies: + decompress-tar: 4.1.1 + file-type: 6.2.0 + is-stream: 1.1.0 + seek-bzip: 1.0.6 + unbzip2-stream: 1.4.3 + dev: false + + /decompress-targz@4.1.1: + resolution: {integrity: sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==} + engines: {node: '>=4'} + dependencies: + decompress-tar: 4.1.1 + file-type: 5.2.0 + is-stream: 1.1.0 + dev: false + + /decompress-unzip@4.0.1: + resolution: {integrity: sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw==} + engines: {node: '>=4'} + dependencies: + file-type: 3.9.0 + get-stream: 2.3.1 + pify: 2.3.0 + yauzl: 2.10.0 + dev: false + + /decompress@4.2.1: + resolution: {integrity: sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ==} + engines: {node: '>=4'} + dependencies: + decompress-tar: 4.1.1 + decompress-tarbz2: 4.1.1 + decompress-targz: 4.1.1 + decompress-unzip: 4.0.1 + graceful-fs: 4.2.11 + make-dir: 1.3.0 + pify: 2.3.0 + strip-dirs: 2.1.0 + dev: false + /dedent@0.7.0: resolution: {integrity: sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==} dev: true @@ -8161,7 +8239,6 @@ packages: resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} dependencies: pend: 1.2.0 - dev: true /feed@4.2.2: resolution: {integrity: sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==} @@ -8211,6 +8288,21 @@ packages: strtok3: 7.0.0 token-types: 5.0.1 + /file-type@3.9.0: + resolution: {integrity: sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==} + engines: {node: '>=0.10.0'} + dev: false + + /file-type@5.2.0: + resolution: {integrity: sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ==} + engines: {node: '>=4'} + dev: false + + /file-type@6.2.0: + resolution: {integrity: sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==} + engines: {node: '>=4'} + dev: false + /filelist@1.0.4: resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==} dependencies: @@ -8679,6 +8771,14 @@ packages: engines: {node: '>=8'} dev: false + /get-stream@2.3.1: + resolution: {integrity: sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA==} + engines: {node: '>=0.10.0'} + dependencies: + object-assign: 4.1.1 + pinkie-promise: 2.0.1 + dev: false + /get-stream@3.0.0: resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==} engines: {node: '>=4'} @@ -9841,6 +9941,10 @@ packages: resolution: {integrity: sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==} dev: false + /is-natural-number@4.0.1: + resolution: {integrity: sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ==} + dev: false + /is-negated-glob@1.0.0: resolution: {integrity: sha512-czXVVn/QEmgvej1f50BZ648vUI+em0xqMq2Sn+QncCLN4zj1UAxlT+kw/6ggQTOaZPd1HqKQGEqbpQVtJucWug==} engines: {node: '>=0.10.0'} @@ -9942,7 +10046,6 @@ packages: /is-stream@1.1.0: resolution: {integrity: sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==} engines: {node: '>=0.10.0'} - dev: true /is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} @@ -12023,6 +12126,13 @@ packages: resolution: {integrity: sha512-3WjL8+ZDouZwKlyJBMp/4LeziLFXgleOdsYu87piGcMLqhBzCsy2QFdbtAwv757TFC/rtqd738fgJw1tFQCSgA==} dev: false + /make-dir@1.3.0: + resolution: {integrity: sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==} + engines: {node: '>=4'} + dependencies: + pify: 3.0.0 + dev: false + /make-dir@3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} engines: {node: '>=8'} @@ -13351,7 +13461,6 @@ packages: /pend@1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} - dev: true /performance-now@2.1.0: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} @@ -13435,7 +13544,11 @@ packages: /pify@2.3.0: resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==} engines: {node: '>=0.10.0'} - dev: true + + /pify@3.0.0: + resolution: {integrity: sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==} + engines: {node: '>=4'} + dev: false /pify@4.0.1: resolution: {integrity: sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==} @@ -13446,12 +13559,10 @@ packages: engines: {node: '>=0.10.0'} dependencies: pinkie: 2.0.4 - dev: true /pinkie@2.0.4: resolution: {integrity: sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==} engines: {node: '>=0.10.0'} - dev: true /pirates@4.0.6: resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==} @@ -14818,6 +14929,13 @@ packages: /seedrandom@3.0.5: resolution: {integrity: sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==} + /seek-bzip@1.0.6: + resolution: {integrity: sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ==} + hasBin: true + dependencies: + commander: 2.20.3 + dev: false + /semver-greatest-satisfied-range@1.1.0: resolution: {integrity: sha512-Ny/iyOzSSa8M5ML46IAx3iXc6tfOsYU2R4AXi2UpHk60Zrgyq6eqPj/xiOfS0rRl/iiQ/rdJkVjw/5cdUyCntQ==} engines: {node: '>= 0.10'} @@ -15450,6 +15568,12 @@ packages: engines: {node: '>=8'} dev: true + /strip-dirs@2.1.0: + resolution: {integrity: sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==} + dependencies: + is-natural-number: 4.0.1 + dev: false + /strip-eof@1.0.0: resolution: {integrity: sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==} engines: {node: '>=0.10.0'} @@ -15643,6 +15767,19 @@ packages: tar-stream: 2.2.0 dev: false + /tar-stream@1.6.2: + resolution: {integrity: sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==} + engines: {node: '>= 0.8.0'} + dependencies: + bl: 1.2.3 + buffer-alloc: 1.2.0 + end-of-stream: 1.4.4 + fs-constants: 1.0.0 + readable-stream: 2.3.8 + to-buffer: 1.1.1 + xtend: 4.0.2 + dev: false + /tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} @@ -15875,6 +16012,10 @@ packages: is-negated-glob: 1.0.0 dev: true + /to-buffer@1.1.1: + resolution: {integrity: sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==} + dev: false + /to-fast-properties@2.0.0: resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} engines: {node: '>=4'} @@ -16413,6 +16554,13 @@ packages: which-boxed-primitive: 1.0.2 dev: true + /unbzip2-stream@1.4.3: + resolution: {integrity: sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==} + dependencies: + buffer: 5.7.1 + through: 2.3.8 + dev: false + /unc-path-regex@0.1.2: resolution: {integrity: sha512-eXL4nmJT7oCpkZsHZUOJo8hcX3GbsiDOa0Qu9F646fi8dT3XuSVopVqAcEiVzSKKH7UoDti23wNX3qGFxcW5Qg==} engines: {node: '>=0.10.0'} @@ -17424,7 +17572,6 @@ packages: dependencies: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 - dev: true /ylru@1.3.2: resolution: {integrity: sha512-RXRJzMiK6U2ye0BlGGZnmpwJDPgakn6aNQ0A7gHRbD4I0uvK4TW6UqkK1V0pp9jskjJBAXd3dRrbzWkqJ+6cxA==} From 32faf81637318eab0f9c513cb7270765322dcd18 Mon Sep 17 00:00:00 2001 From: CGsama Date: Sun, 16 Jul 2023 20:40:40 -0400 Subject: [PATCH 7/7] fix lock --- pnpm-lock.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9a5e94571..0e2c3552e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10745,6 +10745,10 @@ packages: resolution: {integrity: sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==} dev: true + /is-deflate@1.0.0: + resolution: {integrity: sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==} + dev: false + /is-descriptor@0.1.6: resolution: {integrity: sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==} engines: {node: '>=0.10.0'} @@ -10833,16 +10837,14 @@ packages: dependencies: is-extglob: 2.1.1 -<<<<<<< HEAD /is-gzip@1.0.0: resolution: {integrity: sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==} engines: {node: '>=0.10.0'} dev: false -======= + /is-hexadecimal@1.0.4: resolution: {integrity: sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==} dev: true ->>>>>>> 97a0127dbf0b96203902fb075b1d51659a921bfc /is-installed-globally@0.4.0: resolution: {integrity: sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==} @@ -14271,11 +14273,10 @@ packages: resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} dev: false -<<<<<<< HEAD /pako@0.2.9: resolution: {integrity: sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==} dev: false -======= + /paralint@1.2.1: resolution: {integrity: sha512-HPGVDd5eUNYBhtftRypBHlUTMy5UAnHBzFO601oLEODZ0uvXUJge1y43GZQM2CnyOOUyDQlNMK/9vcjjAxbfcA==} hasBin: true @@ -14286,7 +14287,6 @@ packages: minimist: 1.2.8 tslib: 2.6.0 dev: true ->>>>>>> 97a0127dbf0b96203902fb075b1d51659a921bfc /parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}