feat: build app (#112)

* chore: start publish work

* chore: more debug... who overwrote!

permalink: http://whatthecommit.com/2e6bbd4fd1a21e16039ce52216c3c0b4

* update node version requirement (#106)

* update node version requirement

* publish progress

* progress on publish

* style: pretty pretty

* chore: some progress i guess?

* more progress on publish command

* chore: remove magicast

* more progress on command data

* fix: adding extra fields to json output

* fix: stringify again

* rest field cli

* style: run prettier

* chore: friday 5pm

permalink: http://whatthecommit.com/502256b954264d50f29967e1fef8394b

* prettier and more progress on publish

* progress on publishing

* config can be function and publishing seems to be working correctly

* fix, made mistakes

* fix guild command publishing

* fix guild ids not publishing correctly

* edit correctly

* upgrade fire readiers

* refactor and separate some stuff

* ^

* add default_member_permissions

* refator

* refactors

* publish functionality done

* seems like attachment loading works correctly

* extrapolate

* code splitting and faster startup

* fix up bugs

* forgot to merge prompts/plugin

* remove unneeded esbuild plugin for simple define

* build auto generate typings

* fix: template dir after switching commands to dynamic imports

* update preprocessor

* fix: add experimental warning to publish and build

* fix vulnerability

* oops, false alarm

* better sern.build.js config support

* chore: remove pnpm lock

* cleaner build and also some cli options

* fix regression

* add more cli options

* optimize

* no any

---------

Co-authored-by: EvolutionX <evolutionx9777@gmail.com>

package.json

@@ -37,8 +37,10 @@
         "colorette": "2.0.20",
         "commander": "11.0.0",
         "dotenv": "^16.3.1",
+        "esbuild": "^0.19.1",
         "execa": "7.1.1",
         "find-up": "6.3.0",
+        "glob": "^10.3.3",
         "ora": "6.3.1",
         "prompts": "2.4.2",
         "undici": "5.22.1"
@@ -47,7 +49,6 @@
         "@babel/parser": "^7.22.5",
         "@favware/npm-deprecate": "1.0.7",
         "@types/prompts": "2.4.4",
-        "esbuild-plugin-version-injector": "1.1.0",
         "prettier": "2.8.8",
         "tsup": "6.7.0",
         "typescript": "5.1.3"

src/commands/build.ts

@@ -0,0 +1,165 @@
+import esbuild from 'esbuild'
+import { getConfig } from '../utilities/getConfig'
+import { resolve } from 'node:path'
+import { glob } from 'glob'
+import { configDotenv } from 'dotenv'
+import assert from 'node:assert'
+import { imageLoader, validExtensions } from '../plugins/imageLoader'
+import defaultEsbuild from '../utilities/defaultEsbuildConfig'
+import { require } from '../utilities/require'
+import { pathExists, pathExistsSync } from 'find-up'
+import { mkdir, writeFile } from 'fs/promises'
+import * as Preprocessor  from '../utilities/preprocessor'
+import { bold, magentaBright } from 'colorette'
+
+type BuildOptions = {
+    /**
+      * Define __VERSION__
+      * This option is a quick switch to defining the __VERSION__ constant which will be a string of the version provided in 
+      * cwd's package.json
+      */
+    defineVersion?: boolean 
+    /**
+      * default = esm
+      */
+    format?: 'cjs' | 'esm'
+    /** 
+      * extra esbuild plugins to build with sern.
+      */
+    esbuildPlugins?: esbuild.Plugin[]
+    /**
+     * https://esbuild.github.io/api/#drop-labels
+     **/
+    dropLabels?: string[]
+    /**
+     * https://esbuild.github.io/api/#define
+     **/
+    define?: Record<string, string>
+
+    tsconfig?: string;
+    /**
+      * default = 'development'
+      */
+    mode: 'production' | 'development',
+    /**
+      * will search for env file. If none exists, 
+      * default to .env.
+      */
+    env?: string
+}
+
+
+export async function build(options: Record<string,any>) {
+    if(!options.supressWarnings) {
+        console.info(`${magentaBright('EXPERIMENTAL')}: This API has not been stabilized. add -W or --suppress-warnings flag to suppress`)
+    }
+    const sernConfig = await getConfig()
+    let buildConfig: Partial<BuildOptions> = {}; 
+    const entryPoints = await glob(`./src/**/*{${validExtensions.join(',')}}`, {
+                //for some reason, my ignore glob wasn't registering correctly'
+                ignore: {
+                    ignored: p => p.name.endsWith('.d.ts')
+                }
+            })
+    const buildConfigPath = resolve(options.project ?? 'sern.build.js')
+    
+
+    const defaultBuildConfig = {
+        defineVersion: true,
+        format: options.format ?? 'esm',
+        mode: options.mode ?? 'development',
+        dropLabels: [],
+        tsconfig: options.tsconfig ?? resolve('tsconfig.json'),
+        
+        env: options.env ?? resolve('.env')
+    }
+    if(pathExistsSync(buildConfigPath)) {
+        try {
+            buildConfig = {
+                ...defaultBuildConfig,
+                ...(await import('file:///'+buildConfigPath)).default,
+            }
+        } catch(e) {
+            console.log(e)
+            process.exit(1)
+        }
+    } else {
+        buildConfig = {
+            ...defaultBuildConfig
+        }
+        console.log('No build config found, defaulting')
+    }
+    let env = {} as Record<string,string>
+    configDotenv({ path: buildConfig.env, processEnv: env })
+
+    
+    if(env.MODE && !env.NODE_ENV) {
+        console.warn('Use NODE_ENV instead of MODE');
+        console.warn('MODE has no effect.')
+        console.warn(`https://nodejs.dev/en/learn/nodejs-the-difference-between-development-and-production/`);
+    }
+        
+    if(env.NODE_ENV) {
+       buildConfig.mode = env.NODE_ENV as 'production' | 'development'
+       console.log(magentaBright('NODE_ENV:'), 'Found NODE_ENV variable, setting `mode` to this.')
+    }
+    assert(buildConfig.mode === 'development' || buildConfig.mode === 'production', "Mode is not `production` or `development`"); 
+
+    const defaultTsConfig = {
+        extends: "./.sern/tsconfig.json",
+    }
+    !buildConfig.tsconfig && console.log('Using default options for tsconfig', defaultTsConfig);
+    const tsconfigRaw = require(buildConfig.tsconfig!);
+    
+    sernConfig.language === 'typescript' && tsconfigRaw && !tsconfigRaw.extends && (
+        console.warn('tsconfig does not contain an "extends". Will not use sern automatic path aliasing'),
+        console.warn('For projects that predate sern build and want to fully integrate, extend the tsconfig generated in .sern'),
+        console.warn('Extend preexisting tsconfig with top level: "extends": "./.sern/tsconfig.json"')
+    );
+    console.log(bold('Building with:'))
+    console.log(' ', magentaBright('defineVersion'), buildConfig.defineVersion)
+    console.log(' ', magentaBright('format'), buildConfig.format)
+    console.log(' ', magentaBright('mode'), buildConfig.mode)
+    console.log(' ', magentaBright('tsconfig'), buildConfig.tsconfig)
+    console.log(' ', magentaBright('env'), buildConfig.env)
+
+
+    const sernDir = resolve('.sern'),
+          genDir = resolve(sernDir, 'generated'),
+          ambientFilePath = resolve(sernDir, 'ambient.d.ts'),
+          packageJsonPath = resolve('package.json'),
+          sernTsConfigPath = resolve(sernDir, 'tsconfig.json'),
+          packageJson = () => require(packageJsonPath)
+
+
+    if(!(await pathExists(genDir))) {
+        console.log('Making .sern/generated dir, does not exist')
+        await mkdir(genDir) 
+    }
+     
+    try {
+        const defVersion = () => JSON.stringify(packageJson().version)
+        const define = {
+                ...buildConfig.define ?? {},
+                __DEV__: `${buildConfig.mode === 'development'}`,
+                __PROD__: `${buildConfig.mode === 'production'}`,
+        } satisfies Record<string, string>;
+
+        buildConfig.defineVersion && Object.assign(define, { '__VERSION__': defVersion() });
+
+        await Preprocessor.writeTsConfig(buildConfig.format!, sernTsConfigPath, writeFile);
+        await Preprocessor.writeAmbientFile(ambientFilePath, define, writeFile);
+        
+        //https://esbuild.github.io/content-types/#tsconfig-json
+        await esbuild.build({ 
+            entryPoints,
+            plugins: [imageLoader, ...buildConfig.esbuildPlugins??[] ],
+            ...defaultEsbuild(buildConfig.format!, tsconfigRaw),
+            define,
+            dropLabels: [ buildConfig.mode === 'production' ? '__DEV__' : '__PROD__', ...buildConfig.dropLabels!],
+        })
+    } catch(e) {
+        console.error(e)
+        process.exit(1)
+    }
+}

src/commands/help.ts

@@ -7,6 +7,7 @@ export const help = `
  |___/\\___|_|  |_| |_|
 
  Welcome!
+
  If you're new to ${cyanBright('sern')}, run ${magentaBright('npm create @sern/bot')} for an interactive setup to your new bot project!
 
  ${green(`If you have any ideas, suggestions, bug reports, kindly join our support server: https://sern.dev/discord`)}`;

src/commands/publish.ts

@@ -1,12 +1,17 @@
+import { magentaBright } from 'colorette';
 import { getConfig } from '../utilities/getConfig';
 import { fork } from 'node:child_process';
 import { fileURLToPath } from 'url';
 
 export async function publish(commandDir: string | undefined, args: Partial<PublishArgs>) {
+    if(!args.suppressWarnings) {
+        console.info(`${magentaBright('EXPERIMENTAL')}: This API has not been stabilized. add -W or --suppress-warnings flag to suppress`)
+
+    }
     const config = await getConfig();
     // pass in args into the command.
     const rootPath = new URL('../', import.meta.url),
-        publishScript = new URL('./dist/create-publish.js', rootPath);
+        publishScript = new URL('../dist/create-publish.js', rootPath);
     // assign args.import to empty array if non existent
     args.import ??= [];
 
@@ -14,8 +19,8 @@ export async function publish(commandDir: string | undefined, args: Partial<Publ
     args.applicationId && console.info('applicationId passed through command line');
     commandDir && console.info('Publishing with override path: ', commandDir);
 
-    const dotenvLocation = new URL('./node_modules/dotenv/config.js', rootPath),
-        esmLoader = new URL('./node_modules/@esbuild-kit/esm-loader/dist/index.js', rootPath);
+    const dotenvLocation = new URL('../node_modules/dotenv/config.js', rootPath),
+        esmLoader = new URL('../node_modules/@esbuild-kit/esm-loader/dist/index.js', rootPath);
 
     // We dynamically load the create-publish script in a child process so that we can pass the special
     // loader flag to require typescript files
@@ -31,6 +36,7 @@ export async function publish(commandDir: string | undefined, args: Partial<Publ
 }
 
 interface PublishArgs {
+    suppressWarnings: boolean
     import: string[];
     token: string;
     applicationId: string;

src/create-publish.mts

@@ -12,6 +12,7 @@ import type { sernConfig } from './utilities/getConfig';
 import type { PublishableData, PublishableModule, Typeable } from './create-publish.d.ts';
 import { cyanBright, greenBright, redBright } from 'colorette';
 import ora from 'ora';
+import type { TheoreticalEnv } from './types/config';
 
 async function deriveFileInfo(dir: string, file: string) {
     const fullPath = join(dir, file);
@@ -214,6 +215,7 @@ const guildCommandMap = associateGuildIdsWithData(guildedCommands);
 
 let guildCommandMapResponse = new Map<string, Record<string, unknown>>();
 
+
 for (const [guildId, array] of guildCommandMap.entries()) {
     const spin = ora(`[${cyanBright(guildId)}] Updating commands for guild`);
     spin.start();

src/index.ts

@@ -1,52 +1,63 @@
 #!/usr/bin/env node
 
-import { extra } from './commands/extra.js';
-import { help } from './commands/help.js';
-import { init } from './commands/init.js';
-import { publish } from './commands/publish.js';
-
 import { Command } from 'commander';
-import { plugins } from './commands/plugins.js';
 import { yellowBright } from 'colorette';
 export const program = new Command();
 
-const version: string = '[VI]{{inject}}[/VI]';
+const importDynamic = async <T extends string>(filename: T) => import(`./commands/${filename}` as const)
+declare const __VERSION__: string
 
-program //
+program
     .name('sern')
-    .description(help)
-    .version(`sern CLI v${version}`, '-v, --version')
+    .description(await importDynamic('help.js').then(m => m.help))
+    .version(`sern CLI v${__VERSION__}`, '-v, --version')
     .exitOverride(() => process.exit(0));
 
-program //
-    .command(init.name)
-    .description(`Quickest way to scaffold a new project ${yellowBright('[DEPRECATED]')}`)
+program
+    .command('init')
+    .description(
+        `Quickest way to scaffold a new project ${yellowBright('[DEPRECATED]')}`
+    )
     .option('-y', 'Finishes setup as default')
     .option('-s, --sync', 'Syncs the project and generates sern.config.json')
-    .action(init);
+    .action(async (...args) => importDynamic('init.js').then(m => m.init(...args)));
 
-program //
-    .command(plugins.name)
-    .description('Install plugins from https://github.com/sern-handler/awesome-plugins')
+program
+    .command('plugins')
+    .description(
+        'Install plugins from https://github.com/sern-handler/awesome-plugins'
+    )
     .option('-n --name', 'Name of plugin')
-    .action(plugins);
+    .action((...args) => importDynamic('plugins.js').then(m => m.plugins(...args)));
 
-program //
-    .command(extra.name)
+program
+    .command('extra')
     .description('Easy way to add extra things in your sern project')
-    .action(extra);
+    .action((...args) => importDynamic('extra.js').then(m => m.extra(...args)));
 
 program //
     .command('commands')
     .description('Defacto way to manage your slash commands')
     .addCommand(
-        new Command(publish.name)
+        new Command('publish')
             .description('New way to manage your slash commands')
+            .option('-W --suppress-warnings', 'suppress experimental warning')
             .option('-i, --import [scriptPath...]', 'Prerequire a script to load into publisher')
             .option('-t, --token [token]')
             .option('--appId [applicationId]')
             .argument('[path]', 'path with respect to current working directory that will locate all published files')
-            .action(publish)
+            .action(async (...args) => importDynamic('publish.js').then(m => m.publish(...args)))
     );
 
+program 
+    .command('build')
+    .description('Build your bot')
+    .option('-f --format [fmt]', 'The module system of your application. `cjs` or `esm`', 'esm')
+    .option('-m --mode [mode]', 'the mode for sern to build in. `production` or `development`', 'development')
+    .option('-W --suppress-warnings', 'suppress experimental warning')
+    .option('-p --project [filePath]', 'build with this sern.build file')
+    .option('-e --env', 'path to .env file')
+    .option('--tsconfig [filePath]', "Use this tsconfig")
+    .action(async (...args) => importDynamic('build.js').then(m => m.build(...args)))
+
 program.parse();

src/plugins/imageLoader.ts

@@ -0,0 +1,40 @@
+import fs from 'fs/promises'
+import path from 'node:path'
+import { require } from '../utilities/require.js'
+import { type Plugin } from 'esbuild'
+import { basename } from 'node:path'
+
+export const validExtensions = ['.ts','.js', '.json', '.png', '.jpg', '.jpeg', '.webp']
+
+//https://github.com/evanw/esbuild/issues/1051
+export const imageLoader = {
+    name: 'attachment-loader',
+    setup: b => {
+        const filter = new RegExp(`\.${validExtensions.slice(3).join('|')}$`)
+        b.onResolve({ filter }, args => {
+            //if the module is being imported, resolve the path and transform to the js stub
+            if(args.importer) {
+               const newPath = path
+                    .format({ ...path.parse(args.path), base: '', ext: '.js' })
+                    .split(path.sep)
+                    .join(path.posix.sep)
+               return { path: newPath, namespace: 'attachment-loader', external: true }
+            }
+            // if the file is actually the attachment, resolve the full dir 
+            return { path: require.resolve(args.path, { paths: [args.resolveDir] }), namespace: 'attachment-loader' }
+        })       
+
+        b.onLoad({ filter: /.*/, namespace: 'attachment-loader' },
+            async (args) => {
+                const base64 = await fs.readFile(args.path).then(s => s.toString('base64'))
+                return {
+                    contents: `
+                    var __toBuffer = (base64) => Buffer.from(base64, "base64");
+                    module.exports = {
+                        name: '${basename(args.path)}',
+                        attachment: __toBuffer("${base64}") 
+                    }`,
+                }
+            })
+    }
+} satisfies Plugin

src/types/config.d.ts

@@ -0,0 +1,21 @@
+export interface sernConfig {
+    language: 'typescript' | 'javascript';
+    paths: {
+        base: string;
+        commands: string;
+    };
+    scripts?: {
+        prepublish?: string;
+    }
+    buildPath: string;
+    rest?: Record<string, Record<string,unknown>>;
+}
+
+export interface TheoreticalEnv {
+    DISCORD_TOKEN: string
+    APPLICATION_ID: string,
+    MODE: 'PROD' | 'DEV'
+    [name: string]: string
+}
+
+

src/utilities/create.ts

@@ -2,9 +2,7 @@ import { mkdir, readFile, writeFile } from 'fs/promises';
 import { dirname, resolve } from 'node:path';
 import { fileURLToPath, URL } from 'url';
 const root = new URL('../../', import.meta.url);
-const sern = new URL('./@sern/', root);
-const cli = new URL('./cli/', sern);
-const templates = new URL('./templates/', cli);
+const templates = new URL('./templates/', root);
 const extraURL = new URL('./extra/', templates);
 const extraFolder = fileURLToPath(extraURL);
 
@@ -15,6 +13,7 @@ const extraFolder = fileURLToPath(extraURL);
  * @param  location - The location of the file to be created.
  * @param  no_ext - If true, the file will be created without an extension.
  */
+
 export async function create(name: string, lang: string, location: string, no_ext: boolean) {
     const file = `${name}.${lang}.sern`;
 

src/utilities/defaultEsbuildConfig.ts

@@ -0,0 +1,12 @@
+import type esbuild from 'esbuild'
+import { resolve } from 'path'
+
+export default (format: 'cjs' | 'esm', tsconfigRaw: unknown) => ({
+    platform: 'node',
+    format,
+    tsconfigRaw: tsconfigRaw as esbuild.TsconfigRaw,
+    logLevel: 'info',
+    minify: false,
+    outdir: resolve('dist'),
+
+} satisfies esbuild.BuildOptions)

src/utilities/getConfig.ts

@@ -1,3 +1,4 @@
+
 import { readFile } from 'node:fs/promises';
 import { findUp } from 'find-up';
 import assert from 'node:assert';
@@ -20,4 +21,5 @@ export interface sernConfig {
         commands: string;
         events?: string;
     };
+    buildPath: string
 }

src/utilities/preprocessor.ts

@@ -0,0 +1,96 @@
+const declareConstType = (name: string, type: string) => String.raw`declare var ${name}: ${type}`
+
+const processEnvType = (env: NodeJS.ProcessEnv) => {
+    const entries = Object.keys(env)
+
+    const envBuilder = new StringWriter()
+
+    for(const key of entries) {
+        envBuilder.tab()
+        envBuilder.tab()
+        envBuilder.envField(key)
+    }
+    return envBuilder.build()
+}
+
+const determineJSONType = (s : string) => {
+    return typeof JSON.parse(s)
+}
+
+type FileWriter = (path: string, content: string, format: BufferEncoding) => Promise<void>;
+const writeAmbientFile = async (
+    path: string,
+    define: Record<string, string>,
+    writeFile: FileWriter
+) => {
+    const fileContent = new StringWriter()
+    for(const [k,v] of Object.entries(define)) {
+        fileContent.varDecl(k,v)
+    }
+    fileContent
+        .println('declare namespace NodeJS {') 
+        .tab()
+        .println('interface ProcessEnv {')
+        .envFields(process.env)
+        .tab()
+        .println('}')
+        .println('}')
+        
+    await writeFile(path, fileContent.build(), 'utf8')
+}
+
+const writeTsConfig = async (format: 'cjs' | 'esm', configPath: string, fw: FileWriter) => {
+    //maybe better way to do this
+    const target = format === 'esm' ? { target: 'esnext' } : {};   
+    const sernTsConfig = {
+        "compilerOptions": {
+            "moduleResolution": "bundler",
+            "strict": true, 
+            "skipLibCheck": true,
+            ...target,
+            "rootDirs": ["./generated", "../src"]
+        },
+        "include": ["./ambient.d.ts", "../src"]
+    }
+
+    await fw(configPath, JSON.stringify(sernTsConfig, null, 3), 'utf8')
+
+}
+class StringWriter {
+    private fileString = "" 
+
+    tab() {
+        this.fileString+="    "
+        return this;
+    }
+
+    varDecl(name: string, type: string) {
+        this.fileString+=declareConstType(name, determineJSONType(type))+'\n'
+        return this;
+    }
+
+    println(data: string) {
+        this.fileString+=data+"\n"
+        return this;
+    }
+    envField(key: string) {
+        if(/\s|\(|\)/g.test(key)) {
+            this.fileString+=`"${key}": string`
+        } else {
+            this.fileString+=key+ ':'+ 'string'
+        }
+        this.fileString+="\n"
+        return this;
+    }
+
+    envFields(env: NodeJS.ProcessEnv) {
+        this.fileString+=processEnvType(env);
+        return this;
+    }
+    build() {
+        return this.fileString;
+    }
+
+}
+
+export { writeAmbientFile, writeTsConfig }

src/utilities/readPaths.ts

@@ -0,0 +1,50 @@
+import { readdir, stat } from 'fs/promises'
+import { basename, join, extname } from 'path'
+function isSkippable (filename: string) {
+    //empty string is for non extension files (directories)
+    const validExtensions = ['.js', '.cjs', '.mts', '.mjs', '.cts', '.ts', ''];
+    return filename[0] === '!' || !validExtensions.includes(extname(filename));
+}
+
+
+async function deriveFileInfo(dir: string, file: string) {
+    const fullPath = join(dir, file);
+    return {
+        fullPath,
+        fileStats: await stat(fullPath),
+        base: basename(file),
+    };
+}
+
+export async function* readPaths(
+    dir: string,
+    shouldDebug: boolean
+): AsyncGenerator<string> {
+    try {
+        const files = await readdir(dir);
+        for (const file of files) {
+            const { fullPath, fileStats, base } = await deriveFileInfo(
+                dir,
+                file
+            );
+            if (fileStats.isDirectory()) {
+                //Todo: refactor so that i dont repeat myself for files (line 71)
+                if (isSkippable(base)) {
+                    if (shouldDebug)
+                        console.info(`ignored directory: ${fullPath}`);
+                } else {
+                    yield* readPaths(fullPath, shouldDebug);
+                }
+            } else {
+                if (isSkippable(base)) {
+                    if (shouldDebug) console.info(`ignored: ${fullPath}`);
+                } else {
+                    yield 'file:///' + fullPath;
+                }
+            }
+        }
+    } catch (err) {
+        throw err;
+    }
+}
+

src/utilities/require.ts

@@ -0,0 +1,3 @@
+import { createRequire } from 'node:module'
+
+export const require = createRequire(import.meta.url)

tsup.config.ts

@@ -1,7 +1,7 @@
 import { defineConfig } from 'tsup';
-import { esbuildPluginVersionInjector } from 'esbuild-plugin-version-injector';
+import { createRequire } from 'node:module'
 const shared = {
-    entry: ['src/index.ts', 'src/create-publish.mts'],
+    entry: ['src/index.ts', 'src/create-publish.mts', 'src/commands/**', 'sern-tsconfig.json'],
     clean: true,
     sourcemap: true,
 };
@@ -11,8 +11,15 @@ export default defineConfig({
     tsconfig: './tsconfig.json',
     outDir: './dist',
     treeshake: true,
-    esbuildPlugins: [esbuildPluginVersionInjector()],
+    bundle: true,
+    esbuildPlugins: [],
     platform: 'node',
-    splitting: false,
+    splitting: true,
+    define: {
+        __VERSION__: `"${createRequire(import.meta.url)('./package.json').version}"`
+    },
+    loader: {
+        '.json': 'file'
+    },
     ...shared,
 });