'use strict'; const gulp = require('gulp') , fs = require('fs-extra') , semver = require('semver') , formatSize = require(__dirname+'/helpers/files/formatsize.js') , uploadDirectory = require(__dirname+'/helpers/files/uploadDirectory.js') , configs = require(__dirname+'/configs/main.js') , { themes, codeThemes } = require(__dirname+'/helpers/themes.js') , commit = require(__dirname+'/helpers/commit.js') , less = require('gulp-less') , concat = require('gulp-concat') , cleanCSS = require('gulp-clean-css') , uglify = require('gulp-uglify-es').default , del = require('del') , pug = require('pug') , gulppug = require('gulp-pug') , { migrateVersion } = require(__dirname+'/package.json') , { randomBytes } = require('crypto') , paths = { styles: { src: 'gulp/res/css/', dest: 'static/css/' }, images: { src: 'gulp/res/img/*', dest: 'static/file/' }, icons: { src: 'gulp/res/icons/*', dest: 'static/file/' }, scripts: { src: 'gulp/res/js', dest: 'static/js/' }, pug: { src: 'views/', dest: 'static/html/' } }; async function password() { const Mongo = require(__dirname+'/db/db.js') const Redis = require(__dirname+'/redis.js') await Mongo.connect(); const { Accounts } = require(__dirname+'/db/'); const randomPassword = randomBytes(20).toString('base64') await Accounts.changePassword('admin', randomPassword); console.log('=====LOGIN DETAILS=====\nusername: admin\npassword:', randomPassword, '\n======================='); Redis.redisClient.quit(); return Mongo.client.close(); } async function ips() { const Mongo = require(__dirname+'/db/db.js') await Mongo.connect(); const Redis = require(__dirname+'/redis.js') const ipSchedule = require(__dirname+'/schedules/ips.js'); await ipSchedule(); Redis.redisClient.quit(); return Mongo.client.close(); } async function wipe() { const Mongo = require(__dirname+'/db/db.js') const Redis = require(__dirname+'/redis.js') await Mongo.connect(); const db = Mongo.db; const collectionNames = ['accounts', 'bans', 'custompages', 'boards', 'captcha', 'files', 'modlog','news', 'posts', 'poststats', 'ratelimit', 'webring', 'bypass']; for (const name of collectionNames) { //drop collection so gulp reset can be run again. ignores error of dropping non existing collection first time await db.dropCollection(name).catch(e => {}); await db.createCollection(name); } const { Webring, Boards, Posts, Captchas, Ratelimits, News, CustomPages, Accounts, Files, Stats, Modlogs, Bans, Bypass } = require(__dirname+'/db/'); //wipe db shit await Promise.all([ Redis.deletePattern('*'), Captchas.deleteAll(), Ratelimits.deleteAll(), Accounts.deleteAll(), Posts.deleteAll(), Boards.deleteAll(), Webring.deleteAll(), Bans.deleteAll(), Files.deleteAll(), Stats.deleteAll(), Modlogs.deleteAll(), Bypass.deleteAll(), News.deleteAll(), ]); //add indexes - should profiled and changed at some point if necessary await Stats.db.createIndex({board:1, hour:1}) await Boards.db.createIndex({ips: 1, pph:1, sequence_value:1}) await Boards.db.createIndex({'settings.tags':1}) await Boards.db.createIndex({lastPostTimestamp:1}) await Webring.db.createIndex({uniqueUsers:1, postsPerHour:1, totalPosts:1}) await Webring.db.createIndex({tags:1}) await Webring.db.createIndex({lastPostTimestamp:1}) await Bans.db.dropIndexes() await Captchas.db.dropIndexes() await Ratelimits.db.dropIndexes() await Posts.db.dropIndexes() await Modlogs.db.dropIndexes() await CustomPages.db.dropIndexes() await CustomPages.db.createIndex({ 'board': 1, 'page': 1 }, { unique: true }) await Modlogs.db.createIndex({ 'board': 1 }) await Files.db.createIndex({ 'count': 1 }) await Bans.db.createIndex({ 'ip.single': 1 , 'board': 1 }) await Bans.db.createIndex({ 'expireAt': 1 }, { expireAfterSeconds: 0 }) //custom expiry, i.e. it will expire when current date > than this date await Bypass.db.createIndex({ 'expireAt': 1 }, { expireAfterSeconds: 0 }) await Captchas.db.createIndex({ 'expireAt': 1 }, { expireAfterSeconds: 300 }) //captchas valid for 5 minutes await Ratelimits.db.createIndex({ 'expireAt': 1 }, { expireAfterSeconds: 60 }) //per minute captcha ratelimit await Posts.db.createIndex({ 'postId': 1,'board': 1,}) await Posts.db.createIndex({ 'board': 1, 'thread': 1, 'bumped': -1 }) await Posts.db.createIndex({ 'board': 1, 'reports.0': 1 }, { 'partialFilterExpression': { 'reports.0': { '$exists': true } } }) await Posts.db.createIndex({ 'globalreports.0': 1 }, { 'partialFilterExpression': { 'globalreports.0': { '$exists': true } } }) const randomPassword = randomBytes(20).toString('base64') await Accounts.insertOne('admin', 'admin', randomPassword, 0); console.log('=====LOGIN DETAILS=====\nusername: admin\npassword:', randomPassword, '\n======================='); await db.collection('version').replaceOne({ '_id': 'version' }, { '_id': 'version', 'version': migrateVersion }, { upsert: true }); await Mongo.client.close(); Redis.redisClient.quit(); //delete all the static files return Promise.all([ del([ 'static/html/*' ]), del([ 'static/json/*' ]), del([ 'static/banner/*' ]), del([ 'static/captcha/*' ]), del([ 'static/file/*' ]), del([ 'static/css/*' ]), fs.ensureDir(`${uploadDirectory}/captcha`), ]); } //update the css file async function css() { try { //a little more configurable let bypassHeight = (configs.captchaOptions.type === 'google' || configs.captchaOptions.type === 'hcaptcha') ? 500 : configs.captchaOptions.type === 'grid' ? 330 : 235; let captchaHeight = configs.captchaOptions.type === 'text' ? 80 : configs.captchaOptions.type === 'grid' ? configs.captchaOptions.grid.imageSize+30 : 200; //google/hcaptcha doesnt need this set let captchaWidth = configs.captchaOptions.type === 'text' ? 210 : configs.captchaOptions.type === 'grid' ? configs.captchaOptions.grid.imageSize+30 : 200; //google/hcaptcha doesnt need this set const cssLocals = `:root { --attachment-img: url('/file/attachment.png'); --spoiler-img: url('/file/spoiler.png'); --audio-img: url('/file/audio.png'); --thumbnail-size: ${configs.thumbSize}px; --captcha-w: ${captchaWidth}px; --captcha-h: ${captchaHeight}px; --bypass-height: ${bypassHeight}px; }`; fs.writeFileSync('gulp/res/css/locals.css', cssLocals); fs.symlinkSync(__dirname+'/node_modules/highlight.js/styles', __dirname+'/gulp/res/css/codethemes', 'dir'); } catch (e) { if (e.code !== 'EEXIST') { //already exists, ignore error console.log(e); } } await gulp.src([ `${paths.styles.src}/themes/*.css`, ]) .pipe(less()) .pipe(cleanCSS()) .pipe(gulp.dest(`${paths.styles.dest}/themes/`)); await gulp.src([ `${paths.styles.src}/codethemes/*.css`, ]) .pipe(less()) .pipe(cleanCSS()) .pipe(gulp.dest(`${paths.styles.dest}/codethemes/`)); await gulp.src([ `${paths.styles.src}/locals.css`, `${paths.styles.src}/nscaptcha.css`, ]) .pipe(concat('nscaptcha.css')) .pipe(less()) .pipe(cleanCSS()) .pipe(gulp.dest(paths.styles.dest)); return gulp.src([ `${paths.styles.src}/locals.css`, `${paths.styles.src}/style.css`, `${paths.styles.src}/*.css`, `!${paths.styles.src}/nscaptcha.css`, ]) .pipe(concat('style.css')) .pipe(less()) .pipe(cleanCSS()) .pipe(gulp.dest(paths.styles.dest)); } //spoiler/deleted image, default banner, spoiler/sticky/sage/cycle icons function images() { return gulp.src(paths.images.src) .pipe(gulp.dest(paths.images.dest)); } //favicon/safari/chrome/mstiles, etc function icons() { return gulp.src(paths.icons.src) .pipe(gulp.dest(paths.icons.dest)); } async function cache() { const Redis = require(__dirname+'/redis.js') await Promise.all([ Redis.deletePattern('boards:listed'), Redis.deletePattern('board:*'), Redis.deletePattern('boardlist:*'), Redis.deletePattern('banners:*'), Redis.deletePattern('users:*'), Redis.deletePattern('blacklisted:*'), Redis.deletePattern('overboard'), Redis.deletePattern('catalog'), ]); Redis.redisClient.quit(); } function deletehtml() { return del([ 'static/html/*' ]); } function custompages() { return gulp.src([ `${paths.pug.src}/custompages/*.pug`, `${paths.pug.src}/pages/404.pug`, `${paths.pug.src}/pages/500.pug`, `${paths.pug.src}/pages/502.pug`, `${paths.pug.src}/pages/503.pug`, `${paths.pug.src}/pages/504.pug` ]) .pipe(gulppug({ locals: { early404Fraction: configs.early404Fraction, early404Replies: configs.early404Replies, meta: configs.meta, enableWebring: configs.enableWebring, globalLimits: configs.globalLimits, codeLanguages: configs.highlightOptions.languageSubset, defaultTheme: configs.boardDefaults.theme, defaultCodeTheme: configs.boardDefaults.codeTheme, postFilesSize: formatSize(configs.globalLimits.postFilesSize.max), captchaType: configs.captchaOptions.type, googleRecaptchaSiteKey: configs.captchaOptions.google.siteKey, hcaptchaSitekey: configs.captchaOptions.hcaptcha.siteKey, captchaGridSize: configs.captchaOptions.grid.size, commit, } })) .pipe(gulp.dest(paths.pug.dest)); } function scripts() { try { const locals = `const themes = ['${themes.join("', '")}']; const codeThemes = ['${codeThemes.join("', '")}']; const captchaType = '${configs.captchaOptions.type}'; const captchaGridSize = ${configs.captchaOptions.grid.size}; const SERVER_TIMEZONE = '${Intl.DateTimeFormat().resolvedOptions().timeZone}'; const ipHashPermLevel = ${configs.ipHashPermLevel}; const settings = ${JSON.stringify(configs.frontendScriptDefault)}; `; fs.writeFileSync('gulp/res/js/locals.js', locals); fs.writeFileSync('gulp/res/js/post.js', pug.compileFileClient(`${paths.pug.src}/includes/post.pug`, { compileDebug: false, debug: false, name: 'post' })); fs.writeFileSync('gulp/res/js/modal.js', pug.compileFileClient(`${paths.pug.src}/includes/modal.pug`, { compileDebug: false, debug: false, name: 'modal' })); fs.writeFileSync('gulp/res/js/uploaditem.js', pug.compileFileClient(`${paths.pug.src}/includes/uploaditem.pug`, { compileDebug: false, debug: false, name: 'uploaditem' })); fs.writeFileSync('gulp/res/js/pugfilters.js', pug.compileFileClient(`${paths.pug.src}/includes/filters.pug`, { compileDebug: false, debug: false, name: 'filters' })); fs.writeFileSync('gulp/res/js/captchaformsection.js', pug.compileFileClient(`${paths.pug.src}/includes/captchaformsection.pug`, { compileDebug: false, debug: false, name: 'captchaformsection' })); fs.symlinkSync(__dirname+'/node_modules/socket.io-client/dist/socket.io.slim.js', __dirname+'/gulp/res/js/socket.io.js', 'file'); } catch (e) { if (e.code !== 'EEXIST') { console.log(e); } } gulp.src([ //put scripts in order for dependencies `${paths.scripts.src}/locals.js`, `${paths.scripts.src}/localstorage.js`, `${paths.scripts.src}/modal.js`, `${paths.scripts.src}/pugfilters.js`, `${paths.scripts.src}/post.js`, `${paths.scripts.src}/settings.js`, `${paths.scripts.src}/live.js`, `${paths.scripts.src}/captcha.js`, `${paths.scripts.src}/forms.js`, `${paths.scripts.src}/*.js`, `!${paths.scripts.src}/hidefileinput.js`, `!${paths.scripts.src}/dragable.js`, `!${paths.scripts.src}/filters.js`, `!${paths.scripts.src}/hideimages.js`, `!${paths.scripts.src}/yous.js`, `!${paths.scripts.src}/catalog.js`, `!${paths.scripts.src}/time.js`, `!${paths.scripts.src}/themelist.js`, //dont include files from my fuck up with git. todo: make this a migration? `!${paths.scripts.src}/timezone.js`, ]) .pipe(concat('all.js')) .pipe(uglify({compress:false})) .pipe(gulp.dest(paths.scripts.dest)); return gulp.src([ `${paths.scripts.src}/hidefileinput.js`, `${paths.scripts.src}/dragable.js`, `${paths.scripts.src}/hideimages.js`, `${paths.scripts.src}/yous.js`, `${paths.scripts.src}/filters.js`, `${paths.scripts.src}/catalog.js`, `${paths.scripts.src}/time.js`, ]) .pipe(concat('render.js')) .pipe(uglify({compress:false})) .pipe(gulp.dest(paths.scripts.dest)); } async function migrate() { const Mongo = require(__dirname+'/db/db.js') const Redis = require(__dirname+'/redis.js') await Mongo.connect(); const db = Mongo.db; //get current version from db if present (set in 'reset' task in recent versions) let currentVersion = await db.collection('version').findOne({ '_id': 'version' }).then(res => res ? res.version : '0.0.0'); // 0.0.0 for old versions if (semver.lt(currentVersion, migrateVersion)) { console.log(`Current version: ${currentVersion}`); const migrations = require(__dirname+'/migrations/'); const migrationVersions = Object.keys(migrations) .sort(semver.compare) .filter(v => semver.gt(v, currentVersion)); console.log(`Migrations needed: ${currentVersion} -> ${migrationVersions.join(' -> ')}`); for (let ver of migrationVersions) { console.log(`=====\nStarting migration to version ${ver}`); try { await migrations[ver](db, Redis); await db.collection('version').replaceOne({ '_id': 'version' }, { '_id': 'version', 'version': ver }, { upsert: true }); } catch (e) { console.error(e); console.warn(`Migration to ${ver} encountered an error`); } console.log(`Finished migrating to version ${ver}`); } } else { console.log(`Migration not required, you are already on the current version (${migrateVersion})`) } await Mongo.client.close(); Redis.redisClient.quit(); } const build = gulp.parallel(gulp.series(scripts, css), images, icons, gulp.series(deletehtml, custompages)); const reset = gulp.series(wipe, build); const html = gulp.series(deletehtml, custompages); module.exports = { html, css, images, icons, reset, custompages, scripts, wipe, cache, migrate, password, ips, default: build, };