From c58ce6c53bc19ab9fa902b875002c497c60c4112 Mon Sep 17 00:00:00 2001 From: s1idewhist1e Date: Fri, 31 Mar 2023 01:58:28 +0000 Subject: [PATCH] Refactor: use rust for native mastodon id conversion (#9786) This uses [napi-rs](https://napi.rs/) to allow for automatic generation of node bindings for the native code. I also changed the `isolatedModules` TS flag to false to allow for `static enum` to be shared across modules. It doesn't seem to be necessary for the build system that CK uses. Currently this method does not work with ID generators with longer IDs. Likely the best solution is to add another key in the database. Some benchmarks for 1 million conversions: ``` node, x1_000_000: 2.847s rust, x1_000_000: 1.265s ``` There are still optimizations that can be made, but I think this is a good starting point and a good way to bring rust into the CK stack. Co-authored-by: s1idewhist1e Reviewed-on: https://codeberg.org/calckey/calckey/pulls/9786 Co-authored-by: s1idewhist1e Co-committed-by: s1idewhist1e --- README.md | 1 + package.json | 1 + .../backend/native-utils/.cargo/config.toml | 3 + packages/backend/native-utils/.gitignore | 200 ++++++++++++++++++ packages/backend/native-utils/.npmignore | 13 ++ packages/backend/native-utils/Cargo.toml | 18 ++ .../native-utils/__test__/index.spec.mjs | 7 + packages/backend/native-utils/build.rs | 5 + .../npm/android-arm-eabi/README.md | 3 + .../npm/android-arm-eabi/package.json | 18 ++ .../native-utils/npm/android-arm64/README.md | 3 + .../npm/android-arm64/package.json | 18 ++ .../native-utils/npm/darwin-arm64/README.md | 3 + .../npm/darwin-arm64/package.json | 18 ++ .../npm/darwin-universal/README.md | 3 + .../npm/darwin-universal/package.json | 15 ++ .../native-utils/npm/darwin-x64/README.md | 3 + .../native-utils/npm/darwin-x64/package.json | 18 ++ .../native-utils/npm/freebsd-x64/README.md | 3 + .../native-utils/npm/freebsd-x64/package.json | 18 ++ .../npm/linux-arm-gnueabihf/README.md | 3 + .../npm/linux-arm-gnueabihf/package.json | 18 ++ .../npm/linux-arm64-gnu/README.md | 3 + .../npm/linux-arm64-gnu/package.json | 21 ++ .../npm/linux-arm64-musl/README.md | 3 + .../npm/linux-arm64-musl/package.json | 21 ++ .../native-utils/npm/linux-x64-gnu/README.md | 3 + .../npm/linux-x64-gnu/package.json | 21 ++ .../native-utils/npm/linux-x64-musl/README.md | 3 + .../npm/linux-x64-musl/package.json | 21 ++ .../npm/win32-arm64-msvc/README.md | 3 + .../npm/win32-arm64-msvc/package.json | 18 ++ .../npm/win32-ia32-msvc/README.md | 3 + .../npm/win32-ia32-msvc/package.json | 18 ++ .../native-utils/npm/win32-x64-msvc/README.md | 3 + .../npm/win32-x64-msvc/package.json | 18 ++ packages/backend/native-utils/package.json | 44 ++++ packages/backend/native-utils/rustfmt.toml | 2 + packages/backend/native-utils/src/lib.rs | 2 + .../backend/native-utils/src/mastodon_api.rs | 70 ++++++ packages/backend/package.json | 10 +- packages/backend/src/server/api/index.ts | 31 +-- .../server/api/mastodon/endpoints/account.ts | 2 +- packages/backend/tsconfig.json | 2 +- pnpm-lock.yaml | 12 ++ 45 files changed, 692 insertions(+), 34 deletions(-) create mode 100644 packages/backend/native-utils/.cargo/config.toml create mode 100644 packages/backend/native-utils/.gitignore create mode 100644 packages/backend/native-utils/.npmignore create mode 100644 packages/backend/native-utils/Cargo.toml create mode 100644 packages/backend/native-utils/__test__/index.spec.mjs create mode 100644 packages/backend/native-utils/build.rs create mode 100644 packages/backend/native-utils/npm/android-arm-eabi/README.md create mode 100644 packages/backend/native-utils/npm/android-arm-eabi/package.json create mode 100644 packages/backend/native-utils/npm/android-arm64/README.md create mode 100644 packages/backend/native-utils/npm/android-arm64/package.json create mode 100644 packages/backend/native-utils/npm/darwin-arm64/README.md create mode 100644 packages/backend/native-utils/npm/darwin-arm64/package.json create mode 100644 packages/backend/native-utils/npm/darwin-universal/README.md create mode 100644 packages/backend/native-utils/npm/darwin-universal/package.json create mode 100644 packages/backend/native-utils/npm/darwin-x64/README.md create mode 100644 packages/backend/native-utils/npm/darwin-x64/package.json create mode 100644 packages/backend/native-utils/npm/freebsd-x64/README.md create mode 100644 packages/backend/native-utils/npm/freebsd-x64/package.json create mode 100644 packages/backend/native-utils/npm/linux-arm-gnueabihf/README.md create mode 100644 packages/backend/native-utils/npm/linux-arm-gnueabihf/package.json create mode 100644 packages/backend/native-utils/npm/linux-arm64-gnu/README.md create mode 100644 packages/backend/native-utils/npm/linux-arm64-gnu/package.json create mode 100644 packages/backend/native-utils/npm/linux-arm64-musl/README.md create mode 100644 packages/backend/native-utils/npm/linux-arm64-musl/package.json create mode 100644 packages/backend/native-utils/npm/linux-x64-gnu/README.md create mode 100644 packages/backend/native-utils/npm/linux-x64-gnu/package.json create mode 100644 packages/backend/native-utils/npm/linux-x64-musl/README.md create mode 100644 packages/backend/native-utils/npm/linux-x64-musl/package.json create mode 100644 packages/backend/native-utils/npm/win32-arm64-msvc/README.md create mode 100644 packages/backend/native-utils/npm/win32-arm64-msvc/package.json create mode 100644 packages/backend/native-utils/npm/win32-ia32-msvc/README.md create mode 100644 packages/backend/native-utils/npm/win32-ia32-msvc/package.json create mode 100644 packages/backend/native-utils/npm/win32-x64-msvc/README.md create mode 100644 packages/backend/native-utils/npm/win32-x64-msvc/package.json create mode 100644 packages/backend/native-utils/package.json create mode 100644 packages/backend/native-utils/rustfmt.toml create mode 100644 packages/backend/native-utils/src/lib.rs create mode 100644 packages/backend/native-utils/src/mastodon_api.rs diff --git a/README.md b/README.md index 551210676..af117ac4a 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ If you have access to a server that supports one of the sources below, I recomme ### 🏗️ Build dependencies +- 🦀 [Rust toolchain](https://www.rust-lang.org/) - 🦬 C/C++ compiler & build tools - `build-essential` on Debian/Ubuntu Linux - `base-devel` on Arch Linux diff --git a/package.json b/package.json index 42579d772..1c25acdc2 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "dependencies": { "@bull-board/api": "^4.10.2", "@bull-board/ui": "^4.10.2", + "@napi-rs/cli": "^2.15.0", "@tensorflow/tfjs": "^3.21.0", "calckey-js": "^0.0.22", "js-yaml": "4.1.0", diff --git a/packages/backend/native-utils/.cargo/config.toml b/packages/backend/native-utils/.cargo/config.toml new file mode 100644 index 000000000..7ede30ee0 --- /dev/null +++ b/packages/backend/native-utils/.cargo/config.toml @@ -0,0 +1,3 @@ +[target.aarch64-unknown-linux-musl] +linker = "aarch64-linux-musl-gcc" +rustflags = ["-C", "target-feature=-crt-static"] \ No newline at end of file diff --git a/packages/backend/native-utils/.gitignore b/packages/backend/native-utils/.gitignore new file mode 100644 index 000000000..78b75d55a --- /dev/null +++ b/packages/backend/native-utils/.gitignore @@ -0,0 +1,200 @@ +# Created by https://www.toptal.com/developers/gitignore/api/node +# Edit at https://www.toptal.com/developers/gitignore?templates=node + +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test + +# parcel-bundler cache (https://parceljs.org/) +.cache + +# Next.js build output +.next + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# End of https://www.toptal.com/developers/gitignore/api/node + +# Created by https://www.toptal.com/developers/gitignore/api/macos +# Edit at https://www.toptal.com/developers/gitignore?templates=macos + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +# End of https://www.toptal.com/developers/gitignore/api/macos + +# Created by https://www.toptal.com/developers/gitignore/api/windows +# Edit at https://www.toptal.com/developers/gitignore?templates=windows + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/windows + +# napi-rs generated files +built/ + +#Added by cargo + +/target +Cargo.lock + +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions + +*.node diff --git a/packages/backend/native-utils/.npmignore b/packages/backend/native-utils/.npmignore new file mode 100644 index 000000000..ec144db2a --- /dev/null +++ b/packages/backend/native-utils/.npmignore @@ -0,0 +1,13 @@ +target +Cargo.lock +.cargo +.github +npm +.eslintrc +.prettierignore +rustfmt.toml +yarn.lock +*.node +.yarn +__test__ +renovate.json diff --git a/packages/backend/native-utils/Cargo.toml b/packages/backend/native-utils/Cargo.toml new file mode 100644 index 000000000..4f7fb4c39 --- /dev/null +++ b/packages/backend/native-utils/Cargo.toml @@ -0,0 +1,18 @@ +[package] +edition = "2021" +name = "native-utils" +version = "0.0.0" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix +napi = { version = "2.12.0", default-features = false, features = ["napi4"] } +napi-derive = "2.12.0" + +[build-dependencies] +napi-build = "2.0.1" + +[profile.release] +lto = true diff --git a/packages/backend/native-utils/__test__/index.spec.mjs b/packages/backend/native-utils/__test__/index.spec.mjs new file mode 100644 index 000000000..1ade4cafe --- /dev/null +++ b/packages/backend/native-utils/__test__/index.spec.mjs @@ -0,0 +1,7 @@ +import test from 'ava' + +import { sum } from '../index.js' + +test('sum from native', (t) => { + t.is(sum(1, 2), 3) +}) diff --git a/packages/backend/native-utils/build.rs b/packages/backend/native-utils/build.rs new file mode 100644 index 000000000..1f866b6a3 --- /dev/null +++ b/packages/backend/native-utils/build.rs @@ -0,0 +1,5 @@ +extern crate napi_build; + +fn main() { + napi_build::setup(); +} diff --git a/packages/backend/native-utils/npm/android-arm-eabi/README.md b/packages/backend/native-utils/npm/android-arm-eabi/README.md new file mode 100644 index 000000000..10199cb8e --- /dev/null +++ b/packages/backend/native-utils/npm/android-arm-eabi/README.md @@ -0,0 +1,3 @@ +# `native-utils-android-arm-eabi` + +This is the **armv7-linux-androideabi** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/android-arm-eabi/package.json b/packages/backend/native-utils/npm/android-arm-eabi/package.json new file mode 100644 index 000000000..b4404c410 --- /dev/null +++ b/packages/backend/native-utils/npm/android-arm-eabi/package.json @@ -0,0 +1,18 @@ +{ + "name": "native-utils-android-arm-eabi", + "version": "0.0.0", + "os": [ + "android" + ], + "cpu": [ + "arm" + ], + "main": "native-utils.android-arm-eabi.node", + "files": [ + "native-utils.android-arm-eabi.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/android-arm64/README.md b/packages/backend/native-utils/npm/android-arm64/README.md new file mode 100644 index 000000000..c32c2fe71 --- /dev/null +++ b/packages/backend/native-utils/npm/android-arm64/README.md @@ -0,0 +1,3 @@ +# `native-utils-android-arm64` + +This is the **aarch64-linux-android** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/android-arm64/package.json b/packages/backend/native-utils/npm/android-arm64/package.json new file mode 100644 index 000000000..9050ef37b --- /dev/null +++ b/packages/backend/native-utils/npm/android-arm64/package.json @@ -0,0 +1,18 @@ +{ + "name": "native-utils-android-arm64", + "version": "0.0.0", + "os": [ + "android" + ], + "cpu": [ + "arm64" + ], + "main": "native-utils.android-arm64.node", + "files": [ + "native-utils.android-arm64.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/darwin-arm64/README.md b/packages/backend/native-utils/npm/darwin-arm64/README.md new file mode 100644 index 000000000..870390222 --- /dev/null +++ b/packages/backend/native-utils/npm/darwin-arm64/README.md @@ -0,0 +1,3 @@ +# `native-utils-darwin-arm64` + +This is the **aarch64-apple-darwin** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/darwin-arm64/package.json b/packages/backend/native-utils/npm/darwin-arm64/package.json new file mode 100644 index 000000000..a7fcef289 --- /dev/null +++ b/packages/backend/native-utils/npm/darwin-arm64/package.json @@ -0,0 +1,18 @@ +{ + "name": "native-utils-darwin-arm64", + "version": "0.0.0", + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "main": "native-utils.darwin-arm64.node", + "files": [ + "native-utils.darwin-arm64.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/darwin-universal/README.md b/packages/backend/native-utils/npm/darwin-universal/README.md new file mode 100644 index 000000000..098bb3590 --- /dev/null +++ b/packages/backend/native-utils/npm/darwin-universal/README.md @@ -0,0 +1,3 @@ +# `native-utils-darwin-universal` + +This is the **universal-apple-darwin** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/darwin-universal/package.json b/packages/backend/native-utils/npm/darwin-universal/package.json new file mode 100644 index 000000000..a46061d42 --- /dev/null +++ b/packages/backend/native-utils/npm/darwin-universal/package.json @@ -0,0 +1,15 @@ +{ + "name": "native-utils-darwin-universal", + "version": "0.0.0", + "os": [ + "darwin" + ], + "main": "native-utils.darwin-universal.node", + "files": [ + "native-utils.darwin-universal.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/darwin-x64/README.md b/packages/backend/native-utils/npm/darwin-x64/README.md new file mode 100644 index 000000000..0acf36335 --- /dev/null +++ b/packages/backend/native-utils/npm/darwin-x64/README.md @@ -0,0 +1,3 @@ +# `native-utils-darwin-x64` + +This is the **x86_64-apple-darwin** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/darwin-x64/package.json b/packages/backend/native-utils/npm/darwin-x64/package.json new file mode 100644 index 000000000..6bbcf1d23 --- /dev/null +++ b/packages/backend/native-utils/npm/darwin-x64/package.json @@ -0,0 +1,18 @@ +{ + "name": "native-utils-darwin-x64", + "version": "0.0.0", + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "main": "native-utils.darwin-x64.node", + "files": [ + "native-utils.darwin-x64.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/freebsd-x64/README.md b/packages/backend/native-utils/npm/freebsd-x64/README.md new file mode 100644 index 000000000..2b74996de --- /dev/null +++ b/packages/backend/native-utils/npm/freebsd-x64/README.md @@ -0,0 +1,3 @@ +# `native-utils-freebsd-x64` + +This is the **x86_64-unknown-freebsd** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/freebsd-x64/package.json b/packages/backend/native-utils/npm/freebsd-x64/package.json new file mode 100644 index 000000000..654b8abf3 --- /dev/null +++ b/packages/backend/native-utils/npm/freebsd-x64/package.json @@ -0,0 +1,18 @@ +{ + "name": "native-utils-freebsd-x64", + "version": "0.0.0", + "os": [ + "freebsd" + ], + "cpu": [ + "x64" + ], + "main": "native-utils.freebsd-x64.node", + "files": [ + "native-utils.freebsd-x64.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/linux-arm-gnueabihf/README.md b/packages/backend/native-utils/npm/linux-arm-gnueabihf/README.md new file mode 100644 index 000000000..2203036de --- /dev/null +++ b/packages/backend/native-utils/npm/linux-arm-gnueabihf/README.md @@ -0,0 +1,3 @@ +# `native-utils-linux-arm-gnueabihf` + +This is the **armv7-unknown-linux-gnueabihf** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/linux-arm-gnueabihf/package.json b/packages/backend/native-utils/npm/linux-arm-gnueabihf/package.json new file mode 100644 index 000000000..1e206c078 --- /dev/null +++ b/packages/backend/native-utils/npm/linux-arm-gnueabihf/package.json @@ -0,0 +1,18 @@ +{ + "name": "native-utils-linux-arm-gnueabihf", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "arm" + ], + "main": "native-utils.linux-arm-gnueabihf.node", + "files": [ + "native-utils.linux-arm-gnueabihf.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/linux-arm64-gnu/README.md b/packages/backend/native-utils/npm/linux-arm64-gnu/README.md new file mode 100644 index 000000000..ad3a9333f --- /dev/null +++ b/packages/backend/native-utils/npm/linux-arm64-gnu/README.md @@ -0,0 +1,3 @@ +# `native-utils-linux-arm64-gnu` + +This is the **aarch64-unknown-linux-gnu** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/linux-arm64-gnu/package.json b/packages/backend/native-utils/npm/linux-arm64-gnu/package.json new file mode 100644 index 000000000..aa0b2a805 --- /dev/null +++ b/packages/backend/native-utils/npm/linux-arm64-gnu/package.json @@ -0,0 +1,21 @@ +{ + "name": "native-utils-linux-arm64-gnu", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "main": "native-utils.linux-arm64-gnu.node", + "files": [ + "native-utils.linux-arm64-gnu.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "libc": [ + "glibc" + ] +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/linux-arm64-musl/README.md b/packages/backend/native-utils/npm/linux-arm64-musl/README.md new file mode 100644 index 000000000..df282532f --- /dev/null +++ b/packages/backend/native-utils/npm/linux-arm64-musl/README.md @@ -0,0 +1,3 @@ +# `native-utils-linux-arm64-musl` + +This is the **aarch64-unknown-linux-musl** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/linux-arm64-musl/package.json b/packages/backend/native-utils/npm/linux-arm64-musl/package.json new file mode 100644 index 000000000..99e9387ee --- /dev/null +++ b/packages/backend/native-utils/npm/linux-arm64-musl/package.json @@ -0,0 +1,21 @@ +{ + "name": "native-utils-linux-arm64-musl", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "main": "native-utils.linux-arm64-musl.node", + "files": [ + "native-utils.linux-arm64-musl.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "libc": [ + "musl" + ] +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/linux-x64-gnu/README.md b/packages/backend/native-utils/npm/linux-x64-gnu/README.md new file mode 100644 index 000000000..52eea85aa --- /dev/null +++ b/packages/backend/native-utils/npm/linux-x64-gnu/README.md @@ -0,0 +1,3 @@ +# `native-utils-linux-x64-gnu` + +This is the **x86_64-unknown-linux-gnu** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/linux-x64-gnu/package.json b/packages/backend/native-utils/npm/linux-x64-gnu/package.json new file mode 100644 index 000000000..f99a5f664 --- /dev/null +++ b/packages/backend/native-utils/npm/linux-x64-gnu/package.json @@ -0,0 +1,21 @@ +{ + "name": "native-utils-linux-x64-gnu", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "main": "native-utils.linux-x64-gnu.node", + "files": [ + "native-utils.linux-x64-gnu.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "libc": [ + "glibc" + ] +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/linux-x64-musl/README.md b/packages/backend/native-utils/npm/linux-x64-musl/README.md new file mode 100644 index 000000000..6664b2378 --- /dev/null +++ b/packages/backend/native-utils/npm/linux-x64-musl/README.md @@ -0,0 +1,3 @@ +# `native-utils-linux-x64-musl` + +This is the **x86_64-unknown-linux-musl** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/linux-x64-musl/package.json b/packages/backend/native-utils/npm/linux-x64-musl/package.json new file mode 100644 index 000000000..56b520fff --- /dev/null +++ b/packages/backend/native-utils/npm/linux-x64-musl/package.json @@ -0,0 +1,21 @@ +{ + "name": "native-utils-linux-x64-musl", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "main": "native-utils.linux-x64-musl.node", + "files": [ + "native-utils.linux-x64-musl.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "libc": [ + "musl" + ] +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/win32-arm64-msvc/README.md b/packages/backend/native-utils/npm/win32-arm64-msvc/README.md new file mode 100644 index 000000000..7aec7e0a5 --- /dev/null +++ b/packages/backend/native-utils/npm/win32-arm64-msvc/README.md @@ -0,0 +1,3 @@ +# `native-utils-win32-arm64-msvc` + +This is the **aarch64-pc-windows-msvc** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/win32-arm64-msvc/package.json b/packages/backend/native-utils/npm/win32-arm64-msvc/package.json new file mode 100644 index 000000000..865a77105 --- /dev/null +++ b/packages/backend/native-utils/npm/win32-arm64-msvc/package.json @@ -0,0 +1,18 @@ +{ + "name": "native-utils-win32-arm64-msvc", + "version": "0.0.0", + "os": [ + "win32" + ], + "cpu": [ + "arm64" + ], + "main": "native-utils.win32-arm64-msvc.node", + "files": [ + "native-utils.win32-arm64-msvc.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/win32-ia32-msvc/README.md b/packages/backend/native-utils/npm/win32-ia32-msvc/README.md new file mode 100644 index 000000000..690de1975 --- /dev/null +++ b/packages/backend/native-utils/npm/win32-ia32-msvc/README.md @@ -0,0 +1,3 @@ +# `native-utils-win32-ia32-msvc` + +This is the **i686-pc-windows-msvc** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/win32-ia32-msvc/package.json b/packages/backend/native-utils/npm/win32-ia32-msvc/package.json new file mode 100644 index 000000000..994eff12f --- /dev/null +++ b/packages/backend/native-utils/npm/win32-ia32-msvc/package.json @@ -0,0 +1,18 @@ +{ + "name": "native-utils-win32-ia32-msvc", + "version": "0.0.0", + "os": [ + "win32" + ], + "cpu": [ + "ia32" + ], + "main": "native-utils.win32-ia32-msvc.node", + "files": [ + "native-utils.win32-ia32-msvc.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/npm/win32-x64-msvc/README.md b/packages/backend/native-utils/npm/win32-x64-msvc/README.md new file mode 100644 index 000000000..e34a5ff17 --- /dev/null +++ b/packages/backend/native-utils/npm/win32-x64-msvc/README.md @@ -0,0 +1,3 @@ +# `native-utils-win32-x64-msvc` + +This is the **x86_64-pc-windows-msvc** binary for `native-utils` diff --git a/packages/backend/native-utils/npm/win32-x64-msvc/package.json b/packages/backend/native-utils/npm/win32-x64-msvc/package.json new file mode 100644 index 000000000..33b259b13 --- /dev/null +++ b/packages/backend/native-utils/npm/win32-x64-msvc/package.json @@ -0,0 +1,18 @@ +{ + "name": "native-utils-win32-x64-msvc", + "version": "0.0.0", + "os": [ + "win32" + ], + "cpu": [ + "x64" + ], + "main": "native-utils.win32-x64-msvc.node", + "files": [ + "native-utils.win32-x64-msvc.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/packages/backend/native-utils/package.json b/packages/backend/native-utils/package.json new file mode 100644 index 000000000..787d1bd89 --- /dev/null +++ b/packages/backend/native-utils/package.json @@ -0,0 +1,44 @@ +{ + "name": "native-utils", + "version": "0.0.0", + "main": "built/index.js", + "types": "built/index.d.ts", + "napi": { + "name": "native-utils", + "triples": { + "additional": [ + "aarch64-apple-darwin", + "aarch64-linux-android", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "aarch64-pc-windows-msvc", + "armv7-unknown-linux-gnueabihf", + "x86_64-unknown-linux-musl", + "x86_64-unknown-freebsd", + "i686-pc-windows-msvc", + "armv7-linux-androideabi", + "universal-apple-darwin" + ] + } + }, + "license": "MIT", + "devDependencies": { + "@napi-rs/cli": "^2.15.0", + "ava": "^5.1.1" + }, + "ava": { + "timeout": "3m" + }, + "engines": { + "node": ">= 10" + }, + "scripts": { + "artifacts": "napi artifacts", + "build": "napi build --platform --release ./built/", + "build:debug": "napi build --platform", + "prepublishOnly": "napi prepublish -t npm", + "test": "ava", + "universal": "napi universal", + "version": "napi version" + } +} diff --git a/packages/backend/native-utils/rustfmt.toml b/packages/backend/native-utils/rustfmt.toml new file mode 100644 index 000000000..cab5731ed --- /dev/null +++ b/packages/backend/native-utils/rustfmt.toml @@ -0,0 +1,2 @@ +tab_spaces = 2 +edition = "2021" diff --git a/packages/backend/native-utils/src/lib.rs b/packages/backend/native-utils/src/lib.rs new file mode 100644 index 000000000..bc5b9fc7c --- /dev/null +++ b/packages/backend/native-utils/src/lib.rs @@ -0,0 +1,2 @@ + +pub mod mastodon_api; diff --git a/packages/backend/native-utils/src/mastodon_api.rs b/packages/backend/native-utils/src/mastodon_api.rs new file mode 100644 index 000000000..36b4eb984 --- /dev/null +++ b/packages/backend/native-utils/src/mastodon_api.rs @@ -0,0 +1,70 @@ +use napi::{bindgen_prelude::*, Error, Status}; +use napi_derive::napi; + +static CHAR_COLLECTION: &str = "0123456789abcdefghijklmnopqrstuvwxyz"; + +// -- NAPI exports -- + +#[napi] +pub enum IdConvertType { + MastodonId, + CalckeyId, +} + +#[napi] +pub fn convert_id(in_id: String, id_convert_type: IdConvertType) -> napi::Result { + use IdConvertType::*; + match id_convert_type { + MastodonId => { + let mut out: i64 = 0; + for (i, c) in in_id.to_lowercase().chars().rev().enumerate() { + out += num_from_char(c)? as i64 * 36_i64.pow(i as u32); + } + + Ok(out.to_string()) + } + CalckeyId => { + let mut input: i64 = match in_id.parse() { + Ok(s) => s, + Err(_) => { + return Err(Error::new( + Status::InvalidArg, + "Unable to parse ID as MasstodonId", + )) + } + }; + let mut out = String::new(); + + while input != 0 { + out.insert(0, char_from_num((input % 36) as u8)?); + input /= 36; + } + + Ok(out) + } + } +} + +// -- end -- + +#[inline(always)] +fn num_from_char(character: char) -> napi::Result { + for (i, c) in CHAR_COLLECTION.chars().enumerate() { + if c == character { + return Ok(i as u8); + } + } + + Err(Error::new( + Status::InvalidArg, + "Invalid character in parsed base36 id", + )) +} + +#[inline(always)] +fn char_from_num(number: u8) -> napi::Result { + CHAR_COLLECTION + .chars() + .nth(number as usize) + .ok_or(Error::from_status(Status::Unknown)) +} diff --git a/packages/backend/package.json b/packages/backend/package.json index 1c26d32c3..beea30a8a 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -9,7 +9,7 @@ "migrate": "typeorm migration:run -d ormconfig.js", "revertmigration": "typeorm migration:revert -d ormconfig.js", "check:connect": "node ./check_connect.js", - "build": "pnpm swc src -d built -D", + "build": "napi build --platform --release --cargo-cwd native-utils ./native-utils/built/ && pnpm swc src -d built -D", "watch": "pnpm swc src -d built -D -w", "lint": "pnpm rome check \"src/**/*.ts\"", "mocha": "cross-env NODE_ENV=test TS_NODE_FILES=true TS_NODE_TRANSPILE_ONLY=true TS_NODE_PROJECT=\"./test/tsconfig.json\" mocha", @@ -26,6 +26,7 @@ "@bull-board/api": "^4.6.4", "@bull-board/koa": "^4.6.4", "@bull-board/ui": "^4.6.4", + "@calckey/megalodon": "5.1.21", "@discordapp/twemoji": "14.0.2", "@elastic/elasticsearch": "7.17.0", "@koa/cors": "3.4.3", @@ -38,12 +39,11 @@ "@tensorflow/tfjs": "^4.2.0", "ajv": "8.11.2", "archiver": "5.3.1", - "koa-body": "^6.0.1", "autobind-decorator": "2.4.0", "autolinker": "4.0.0", - "axios": "^1.3.2", "autwh": "0.1.0", "aws-sdk": "2.1277.0", + "axios": "^1.3.2", "bcryptjs": "2.4.3", "blurhash": "1.1.5", "bull": "4.10.2", @@ -72,12 +72,13 @@ "jsonld": "6.0.0", "jsrsasign": "10.6.1", "koa": "2.13.4", - "koa-remove-trailing-slashes": "2.0.3", + "koa-body": "^6.0.1", "koa-bodyparser": "4.3.0", "koa-favicon": "2.1.0", "koa-json-body": "5.3.0", "koa-logger": "3.2.1", "koa-mount": "4.0.0", + "koa-remove-trailing-slashes": "2.0.3", "koa-send": "5.0.1", "koa-slow": "2.1.0", "koa-views": "7.0.2", @@ -85,6 +86,7 @@ "mfm-js": "0.23.2", "mime-types": "2.1.35", "multer": "1.4.4-lts.1", + "native-utils": "link:native-utils", "nested-property": "4.0.0", "node-fetch": "3.3.0", "nodemailer": "6.8.0", diff --git a/packages/backend/src/server/api/index.ts b/packages/backend/src/server/api/index.ts index 4eb87a614..a0833eb3e 100644 --- a/packages/backend/src/server/api/index.ts +++ b/packages/backend/src/server/api/index.ts @@ -21,35 +21,10 @@ import discord from "./service/discord.js"; import github from "./service/github.js"; import twitter from "./service/twitter.js"; import { koaBody } from "koa-body"; +import { convertId, IdConvertType as IdType } from "native-utils" -export enum IdType { - CalckeyId, - MastodonId -}; - -export function convertId(idIn: string, idConvertTo: IdType ) { - let idArray = [] - switch (idConvertTo) { - case IdType.MastodonId: - idArray = [...idIn].map(item => item.charCodeAt(0)); - idArray = idArray.map(item => { - if (item.toString().length < 3) { - return `0${item.toString()}` - } - else return item.toString() - }); - return idArray.join(''); - case IdType.CalckeyId: - for (let i = 0; i < idIn.length; i += 3) { - if ((idIn.length % 3) !== 0) { - idIn = `0${idIn}` - } - idArray.push(idIn.slice(i, i+3)); - } - idArray = idArray.map(item => String.fromCharCode(item)); - return idArray.join(''); - } -}; +// re-export native rust id conversion (function and enum) +export { IdType, convertId }; // Init app const app = new Koa(); diff --git a/packages/backend/src/server/api/mastodon/endpoints/account.ts b/packages/backend/src/server/api/mastodon/endpoints/account.ts index 9f4070f2d..fa218305c 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/account.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/account.ts @@ -89,7 +89,7 @@ export function apiAccountMastodon(router: Router): void { } }); router.get<{ Params: { id: string } }>( - "/v1/accounts/:id(^.*\\d.*$)", + "/v1/accounts/:id", async (ctx) => { const BASE_URL = `${ctx.protocol}://${ctx.hostname}`; const accessTokens = ctx.headers.authorization; diff --git a/packages/backend/tsconfig.json b/packages/backend/tsconfig.json index dea4eb27d..692d7b95b 100644 --- a/packages/backend/tsconfig.json +++ b/packages/backend/tsconfig.json @@ -21,7 +21,7 @@ "experimentalDecorators": true, "emitDecoratorMetadata": true, "resolveJsonModule": true, - "isolatedModules": true, + "isolatedModules": false, "rootDir": "./src", "baseUrl": "./", "paths": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c817abb81..945197b4f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,6 +13,9 @@ importers: '@bull-board/ui': specifier: ^4.10.2 version: 4.10.2 + '@napi-rs/cli': + specifier: ^2.15.0 + version: 2.15.0 '@tensorflow/tfjs': specifier: ^3.21.0 version: 3.21.0(seedrandom@3.0.5) @@ -257,6 +260,9 @@ importers: multer: specifier: 1.4.4-lts.1 version: 1.4.4-lts.1 + native-utils: + specifier: link:native-utils + version: link:native-utils nested-property: specifier: 4.0.0 version: 4.0.0 @@ -1637,6 +1643,12 @@ packages: dev: false optional: true + /@napi-rs/cli@2.15.0: + resolution: {integrity: sha512-RDDr7ZF0cgbd37+NBGeQOjP7Tm/iNM+y3FmrT5bVQBXLePOTuKVC/dBsdN5UZv3Sl2XAwEvBfaGR90E0d8AA6g==} + engines: {node: '>= 10'} + hasBin: true + dev: false + /@nodelib/fs.scandir@2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'}