From 67e7b538603f1c665f8e69fedff50f9e8907e3c3 Mon Sep 17 00:00:00 2001 From: Izan Gil <66965250+SrIzan10@users.noreply.github.com> Date: Sun, 5 Jan 2025 20:50:03 +0100 Subject: [PATCH] feat: devserver safebooru pull and ui fixes --- package.json | 2 + src/app/(public)/page.tsx | 35 +++++++++------ src/app/(public)/post/[id]/page.tsx | 8 +++- src/instrumentation.ts | 63 +++++++++++++++++++++++++++ yarn.lock | 67 +++++++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 15 deletions(-) create mode 100644 src/instrumentation.ts diff --git a/package.json b/package.json index c846d99..e5438e4 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,8 @@ "@radix-ui/react-slot": "^1.1.1", "class-variance-authority": "^0.7.1", "clsx": "^2.1.0", + "cron": "^3.3.2", + "glob": "^11.0.0", "lucia": "^3.1.1", "lucide-react": "^0.368.0", "next": "^15.1.2", diff --git a/src/app/(public)/page.tsx b/src/app/(public)/page.tsx index 8b20fbb..f9ca20f 100644 --- a/src/app/(public)/page.tsx +++ b/src/app/(public)/page.tsx @@ -11,20 +11,29 @@ export default async function Home() {

(very unstable and not feature complete!)

-
+ {Boolean(process.env.SAFEBOORU_PULL) && ( +

+ Development instance is pulling the first 30 safebooru images it finds for demonstration + purposes. +

+ )} +
{posts.map((post) => ( -
- - {''} - +
+ {''} +
))}
diff --git a/src/app/(public)/post/[id]/page.tsx b/src/app/(public)/post/[id]/page.tsx index 782581e..7a61d2d 100644 --- a/src/app/(public)/post/[id]/page.tsx +++ b/src/app/(public)/post/[id]/page.tsx @@ -71,9 +71,13 @@ export default async function Page({ params }: { params: Promise<{ id: string }>

Tags

-
+
{post.tags.map((tag) => ( - + {tag} ))} diff --git a/src/instrumentation.ts b/src/instrumentation.ts new file mode 100644 index 0000000..09f789d --- /dev/null +++ b/src/instrumentation.ts @@ -0,0 +1,63 @@ +import prisma from './lib/db'; + +export async function register() { + if (process.env.SAFEBOORU_PULL !== 'true') return; + if (process.env.NEXT_RUNTIME === 'nodejs') { + const { CronJob } = await import('cron'); + const crypto = await import('crypto'); + const { generateId } = await import('lucia'); + const fs = await import('fs/promises'); + const { default: hashImage } = await import('@/lib/hashImage'); + const { glob } = await import('glob'); + + const job = async () => { + console.log('Deleting prior safebooru posts and accounts...'); + await prisma.post.deleteMany({ + where: { author: { username: { startsWith: 'safebooru-' } } }, + }); + await prisma.user.deleteMany({ where: { username: { startsWith: 'safebooru-' } } }); + const files = await glob('public/uploads/safebooru-*.jpg'); + await Promise.all(files.map((file) => fs.rm(file, { force: true }))); + + console.log('Pulling safebooru images...'); + console.log('Fetching...'); + const res = await fetch( + 'https://safebooru.org/index.php?page=dapi&s=post&q=index&json=1&limit=30', + { headers: { 'User-Agent': 'nextbooru' } } + ); + const posts = await res.json(); + + const genId = generateId(6); + + console.log('Creating account...'); + const account = await prisma.user.create({ + data: { + username: `safebooru-${genId}`, + hashed_password: crypto.randomUUID(), + }, + }); + + console.log('Downloading all images...'); + for (const post of posts) { + const imageUrl = await fetch(post.file_url).then((res) => res.arrayBuffer()); + const savedFilename = `public/uploads/safebooru-${genId}-${post.id}.jpg`; + await fs.writeFile(savedFilename, new Uint8Array(imageUrl)); + const previewHash = await hashImage(Buffer.from(imageUrl)); + await prisma.post.create({ + data: { + imageUrl: savedFilename.replace('public', ''), + previewHash, + authorId: account.id, + tags: post.tags.split(' '), + caption: post.source, + }, + }); + console.log(`Downloaded id ${post.id}`); + } + }; + + await job(); + + new CronJob('0 */2 * * *', async () => await job(), null, true); + } +} diff --git a/yarn.lock b/yarn.lock index f340608..4999efc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1290,6 +1290,11 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/luxon@~3.4.0": + version "3.4.2" + resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-3.4.2.tgz#e4fc7214a420173cea47739c33cdf10874694db7" + integrity sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA== + "@types/node@^20": version "20.12.7" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.7.tgz#04080362fa3dd6c5822061aa3124f5c152cff384" @@ -1814,6 +1819,14 @@ cosmiconfig@^8.1.3: parse-json "^5.2.0" path-type "^4.0.0" +cron@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/cron/-/cron-3.3.2.tgz#d98843f1db6d4cfcf3a4cb028fbc71231c5261f1" + integrity sha512-7o2PH9vKRd4PxB8c2GsHRozfHYT+gIhZG0DI+vzGOdWo42mofO/ooYnyU0CCh27aKzCrUKMAwAwi7xJ84xKSug== + dependencies: + "@types/luxon" "~3.4.0" + luxon "~3.5.0" + cross-spawn@^7.0.0, cross-spawn@^7.0.2: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -2601,6 +2614,18 @@ glob@^10.3.10: minipass "^7.0.4" path-scurry "^1.10.2" +glob@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-11.0.0.tgz#6031df0d7b65eaa1ccb9b29b5ced16cea658e77e" + integrity sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g== + dependencies: + foreground-child "^3.1.0" + jackspeak "^4.0.1" + minimatch "^10.0.0" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^2.0.0" + glob@^7.1.3: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" @@ -3014,6 +3039,13 @@ jackspeak@^2.3.5, jackspeak@^2.3.6: optionalDependencies: "@pkgjs/parseargs" "^0.11.0" +jackspeak@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-4.0.2.tgz#11f9468a3730c6ff6f56823a820d7e3be9bef015" + integrity sha512-bZsjR/iRjl1Nk1UkjGpAzLNfQtzuijhn2g+pbZb98HQ1Gk8vM9hfbxeMBP+M2/UUdwj0RqGG3mlvk2MsAqwvEw== + dependencies: + "@isaacs/cliui" "^8.0.2" + jiti@^1.21.0: version "1.21.0" resolved "https://registry.yarnpkg.com/jiti/-/jiti-1.21.0.tgz#7c97f8fe045724e136a397f7340475244156105d" @@ -3191,6 +3223,11 @@ lru-cache@^10.2.0: resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.2.1.tgz#e8d901141f22937968e45a6533d52824070151e4" integrity sha512-tS24spDe/zXhWbNPErCHs/AGOzbKGHT+ybSBqmdLm8WZ1xXLWvH8Qn71QPAlqVhd0qUTWjy+Kl9JmISgDdEjsA== +lru-cache@^11.0.0: + version "11.0.2" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-11.0.2.tgz#fbd8e7cf8211f5e7e5d91905c415a3f55755ca39" + integrity sha512-123qHRfJBmo2jXDbo/a5YOQrJoHF/GNQTLzQ5+IdK5pWpceK17yRc6ozlWd25FxvGKQbIUs91fDFkXmDHTKcyA== + lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" @@ -3217,6 +3254,11 @@ lucide-react@^0.368.0: resolved "https://registry.yarnpkg.com/lucide-react/-/lucide-react-0.368.0.tgz#3c0ee63f4f7d30ae63b621b2b8f04f9e409ee6e7" integrity sha512-soryVrCjheZs8rbXKdINw9B8iPi5OajBJZMJ1HORig89ljcOcEokKKAgGbg3QWxSXel7JwHOfDFUdDHAKyUAMw== +luxon@~3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.5.0.tgz#6b6f65c5cd1d61d1fd19dbf07ee87a50bf4b8e20" + integrity sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ== + memfs-browser@^3.4.13000: version "3.5.10302" resolved "https://registry.yarnpkg.com/memfs-browser/-/memfs-browser-3.5.10302.tgz#2067baf616a1b3a8e8023a033e5ead434a7ea0c0" @@ -3266,6 +3308,13 @@ minimatch@9.0.3: dependencies: brace-expansion "^2.0.1" +minimatch@^10.0.0: + version "10.0.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.0.1.tgz#ce0521856b453c86e25f2c4c0d03e6ff7ddc440b" + integrity sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ== + dependencies: + brace-expansion "^2.0.1" + minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -3297,6 +3346,11 @@ minimist@^1.2.0, minimist@^1.2.6: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== +minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + mkdirp@^2.1.6: version "2.1.6" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-2.1.6.tgz#964fbcb12b2d8c5d6fbc62a963ac95a273e2cc19" @@ -3536,6 +3590,11 @@ p-locate@^5.0.0: dependencies: p-limit "^3.0.2" +package-json-from-dist@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz#4f1471a010827a86f94cfd9b0727e36d267de505" + integrity sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw== + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -3591,6 +3650,14 @@ path-scurry@^1.10.1, path-scurry@^1.10.2: lru-cache "^10.2.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" +path-scurry@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-2.0.0.tgz#9f052289f23ad8bf9397a2a0425e7b8615c58580" + integrity sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg== + dependencies: + lru-cache "^11.0.0" + minipass "^7.1.2" + path-type@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b"