From 3887ce05da9596200e4ee415580b6093994e2ad2 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Tue, 3 Dec 2024 13:04:12 -0500 Subject: [PATCH] dev: de-couple legacy mkdir and remove it A legacy mkdir function was still being used by the generate_system_fsentries function. However, it wasn't necessary because we now associate fsentries by uuid, eliminating the requirement to create the home directory separately from its subdirectories. The old mkdir function was removed, and generate_system_fsentries was moved to a new service called UserService, which is intended to eventually hold all methods related creating, removing, and updating users. --- mods/mods_available/kdmod/ShareTestService.js | 4 +- src/backend/src/CoreModule.js | 3 + src/backend/src/helpers.js | 291 ------------------ .../modules/selfhosted/DefaultUserService.js | 5 +- src/backend/src/routers/signup.js | 3 +- src/backend/src/services/UserService.js | 111 +++++++ 6 files changed, 121 insertions(+), 296 deletions(-) create mode 100644 src/backend/src/services/UserService.js diff --git a/mods/mods_available/kdmod/ShareTestService.js b/mods/mods_available/kdmod/ShareTestService.js index 3e8e61af..94bf87be 100644 --- a/mods/mods_available/kdmod/ShareTestService.js +++ b/mods/mods_available/kdmod/ShareTestService.js @@ -21,7 +21,6 @@ // we have these things registered in "useapi". const { get_user, - generate_system_fsentries, invalidate_cached_user, deleteUser, } = require('../../../src/backend/src/helpers.js'); @@ -146,7 +145,8 @@ class ShareTestService extends use.Service { ], ); const user = await get_user({ username }); - await generate_system_fsentries(user); + const svc_user = this.services.get('user'); + await svc_user.generate_default_fsentries({ user }); invalidate_cached_user(user); return user; } diff --git a/src/backend/src/CoreModule.js b/src/backend/src/CoreModule.js index 7a4fb769..a482892e 100644 --- a/src/backend/src/CoreModule.js +++ b/src/backend/src/CoreModule.js @@ -351,6 +351,9 @@ const install = async ({ services, app, useapi, modapi }) => { const { ReferralCodeService } = require('./services/ReferralCodeService'); services.registerService('referral-code', ReferralCodeService); + + const { UserService } = require('./services/UserService'); + services.registerService('user', UserService); } const install_legacy = async ({ services }) => { diff --git a/src/backend/src/helpers.js b/src/backend/src/helpers.js index c9de4a8d..0fe8a829 100644 --- a/src/backend/src/helpers.js +++ b/src/backend/src/helpers.js @@ -1155,201 +1155,6 @@ async function jwt_auth(req){ return ancestors; } -// THIS LEGACY FUNCTION IS STILL IN USE -// by: generate_system_fsentries -// TODO: migrate generate_system_fsentries to use QuickMkdir -async function mkdir(options){ - const fs = systemfs; - - debugger; - - const resolved_path = PathBuilder.resolve(options.path, { puterfs: true }); - - const dirpath = _path.dirname(resolved_path); - let target_name = _path.basename(resolved_path); - const overwrite = options.overwrite ?? false; - const dedupe_name = options.dedupe_name ?? false; - const immutable = options.immutable ?? false; - const return_id = options.return_id ?? false; - const no_perm_check = options.no_perm_check ?? false; - - // make parent directories as needed - const create_missing_parents = options.create_missing_parents ?? false; - - // hold a list of all parent directories created in the process - let parent_dirs_created = []; - let overwritten_uid; - - // target_name validation - try{ - validate_fsentry_name(target_name) - }catch(e){ - throw e.message; - } - - // resolve dirpath to its fsentry - let parent = await convert_path_to_fsentry(dirpath); - - // dirpath not found - if(parent === false && !create_missing_parents) - throw new Error("Target path not found"); - // create missing parent directories - else if(parent === false && create_missing_parents){ - const dirs = _path.resolve('/', dirpath).split('/'); - let cur_path = ''; - for(let j=0; j < dirs.length; j++){ - if(dirs[j] === '') - continue; - - cur_path += '/'+dirs[j]; - // skip creating '/[username]' - if(j === 1) - continue; - try{ - let d = await mkdir(fs, {path: cur_path, user: options.user}); - d.path = cur_path; - parent_dirs_created.push(d); - }catch(e){ - console.log(`Skipped mkdir ${cur_path}`); - } - } - // try setting parent again - parent = await convert_path_to_fsentry(dirpath); - if(parent === false) - throw new Error("Target path not found"); - } - - // check permission - if(!no_perm_check && !await chkperm(parent, options.user.id, 'write')) - throw { code:`forbidden`, message: `permission denied.`}; - - // check if a fsentry with the same name exists under this path - const existing_fsentry = await convert_path_to_fsentry(_path.resolve('/', dirpath + '/' + target_name )); - - /** @type BaseDatabaseAccessService */ - const db = services.get('database').get(DB_WRITE, 'filesystem'); - - // if trying to create a directory with an existing path and overwrite==false, throw an error - if(!overwrite && !dedupe_name && existing_fsentry !== false){ - throw { - code: 'path_exists', - message:"A file/directory with the same path already exists.", - entry_name: existing_fsentry.name, - existing_fsentry: { - name: existing_fsentry.name, - uid: existing_fsentry.uuid, - } - }; - } - else if(overwrite && existing_fsentry){ - overwritten_uid = existing_fsentry.uuid; - // check permission - if(!await chkperm(existing_fsentry, options.user.id, 'write')) - throw {code:`forbidden`, message: `permission denied.`}; - // delete existing dir - await db.write( - `DELETE FROM fsentries WHERE id = ? AND user_id = ?`, - [ - //parent_uid - existing_fsentry.uuid, - //user_id - options.user.id, - ]); - } - // dedupe name, generate a new name until its unique - else if(dedupe_name && existing_fsentry !== false){ - for( let i = 1; ; i++){ - let try_new_name = existing_fsentry.name + ' (' + i + ')'; - let check_dupe = await db.read( - "SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1", - [existing_fsentry.parent_uid, try_new_name] - ); - if(check_dupe[0] === undefined){ - target_name = try_new_name; - break; - } - } - } - - // shrotcut? - let shortcut_fsentry; - if(options.shortcut_to){ - shortcut_fsentry = await uuid2fsentry(options.shortcut_to); - if(shortcut_fsentry === false){ - throw ({ code:`not_found`, message: `shortcut_to not found.`}) - }else if(!parent.is_dir){ - throw ({ code:`not_dir`, message: `parent of shortcut_to must be a directory`}) - }else if(!await chkperm(shortcut_fsentry, options.user.id, 'read')){ - throw ({ code:`forbidden`, message: `shortcut_to permission denied.`}) - } - } - - // current epoch - const ts = Math.round(Date.now() / 1000) - const uid = uuidv4(); - - // record in db - let user_id = (parent === null ? options.user.id : parent.user_id); - const { insertId: mkdir_db_id } = await db.write( - `INSERT INTO fsentries - (uuid, parent_uid, user_id, name, is_dir, created, modified, immutable, shortcut_to, is_shortcut) VALUES - ( ?, ?, ?, ?, true, ?, ?, ?, ?, ?)`, - [ - //uuid - uid, - //parent_uid - (parent === null) ? null : parent.uuid, - //user_id - user_id, - //name - target_name, - //created - ts, - //modified - ts, - //immutable - immutable, - //shortcut_to, - shortcut_fsentry ? shortcut_fsentry.id : null, - //is_shortcut, - shortcut_fsentry ? 1 : 0, - ] - ); - - const ret_obj = { - uid : uid, - name: target_name, - immutable: immutable, - is_dir: true, - path: options.path ?? false, - dirpath: dirpath, - is_shared: await is_shared_with_anyone(mkdir_db_id), - overwritten_uid: overwritten_uid, - shortcut_to: shortcut_fsentry ? shortcut_fsentry.uuid : null, - shortcut_to_path: shortcut_fsentry ? await id2path(shortcut_fsentry.id) : null, - parent_dirs_created: parent_dirs_created, - original_client_socket_id: options.original_client_socket_id, - }; - // add existing_fsentry if exists - if(existing_fsentry){ - ret_obj.existing_fsentry ={ - name: existing_fsentry.name, - uid: existing_fsentry.uuid, - } - } - - if(return_id) - ret_obj.id = mkdir_db_id; - - // send realtime success msg to client - let socketio = require('./socketio.js').getio(); - if(socketio){ - socketio.to(user_id).emit('item.added', ret_obj) - } - - return ret_obj; -} - function is_valid_uuid ( uuid ) { let s = "" + uuid; s = s.match(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i); @@ -1412,100 +1217,6 @@ async function app_name_exists(name){ return true; } - -// generates all the default files and directories a user needs, -// generally used for a brand new account -async function generate_system_fsentries(user){ - /** @type BaseDatabaseAccessService */ - const db = services.get('database').get(DB_WRITE, 'filesystem'); - - //------------------------------------------------------------- - // create root `/[username]/` - //------------------------------------------------------------- - const root_dir = await mkdir({ - path: '/' + user.username, - user: user, - immutable: true, - no_perm_check: true, - return_id: true, - }); - - // Normally, it is recommended to use mkdir() to create new folders, - // but during signup this could result in multiple queries to the DB server - // and for servers in remote regions such as Asia this could result in a - // very long time for /signup to finish, sometimes up to 30-40 seconds! - // by combining as many queries as we can into one and avoiding multiple back-and-forth - // with the DB server, we can speed this process up significantly. - const ts = Date.now()/1000; - - // Generate UUIDs for all the default folders and files - let trash_uuid = uuidv4(); - let appdata_uuid = uuidv4(); - let desktop_uuid = uuidv4(); - let documents_uuid = uuidv4(); - let pictures_uuid = uuidv4(); - let videos_uuid = uuidv4(); - let public_uuid = uuidv4(); - - const insert_res = await db.write( - `INSERT INTO fsentries - (uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable) VALUES - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true) - `, - [ - // Trash - trash_uuid, root_dir.uid, user.id, 'Trash', `/${user.username}/Trash`, ts, ts, - // AppData - appdata_uuid, root_dir.uid, user.id, 'AppData', `/${user.username}/AppData`, ts, ts, - // Desktop - desktop_uuid, root_dir.uid, user.id, 'Desktop', `/${user.username}/Desktop`, ts, ts, - // Documents - documents_uuid, root_dir.uid, user.id, 'Documents', `/${user.username}/Documents`, ts, ts, - // Pictures - pictures_uuid, root_dir.uid, user.id, 'Pictures', `/${user.username}/Pictures`, ts, ts, - // Videos - videos_uuid, root_dir.uid, user.id, 'Videos', `/${user.username}/Videos`, ts, ts, - // Public - public_uuid, root_dir.uid, user.id, 'Public', `/${user.username}/Public`, ts, ts, - ] - ); - - // https://stackoverflow.com/a/50103616 - let trash_id = insert_res.insertId; - let appdata_id = insert_res.insertId + 1; - let desktop_id = insert_res.insertId + 2; - let documents_id = insert_res.insertId + 3; - let pictures_id = insert_res.insertId + 4; - let videos_id = insert_res.insertId + 5; - let public_id = insert_res.insertId + 6; - - // Asynchronously set the user's system folders uuids in database - // This is for caching purposes, so we don't have to query the DB every time we need to access these folders - // This is also possible because we know the user's system folders uuids will never change - - // TODO: pass to IIAFE manager to avoid unhandled promise rejection - // (IIAFE manager doesn't exist yet, hence this is a TODO) - db.write( - `UPDATE user SET - trash_uuid=?, appdata_uuid=?, desktop_uuid=?, documents_uuid=?, pictures_uuid=?, videos_uuid=?, public_uuid=?, - trash_id=?, appdata_id=?, desktop_id=?, documents_id=?, pictures_id=?, videos_id=?, public_id=? - - WHERE id=?`, - [ - trash_uuid, appdata_uuid, desktop_uuid, documents_uuid, pictures_uuid, videos_uuid, public_uuid, - trash_id, appdata_id, desktop_id, documents_id, pictures_id, videos_id, public_id, - user.id - ] - ); - invalidate_cached_user(user); -} - function send_email_verification_code(email_confirm_code, email){ const svc_email = Context.get('services').get('email'); svc_email.send_email({ email }, 'email_verification_code', { @@ -1905,7 +1616,6 @@ module.exports = { gen_public_token, get_taskbar_items, get_url_from_req, - generate_system_fsentries, generate_random_str, generate_random_username, get_app, @@ -1926,7 +1636,6 @@ module.exports = { is_specifically_uuidv4, is_valid_url, jwt_auth, - mkdir, mv, number_format, refresh_apps_cache, diff --git a/src/backend/src/modules/selfhosted/DefaultUserService.js b/src/backend/src/modules/selfhosted/DefaultUserService.js index db1c5d08..4d032dce 100644 --- a/src/backend/src/modules/selfhosted/DefaultUserService.js +++ b/src/backend/src/modules/selfhosted/DefaultUserService.js @@ -20,7 +20,7 @@ const { QuickMkdir } = require("../../filesystem/hl_operations/hl_mkdir"); const { HLWrite } = require("../../filesystem/hl_operations/hl_write"); const { NodePathSelector } = require("../../filesystem/node/selectors"); const { surrounding_box } = require("../../fun/dev-console-ui-utils"); -const { get_user, generate_system_fsentries, invalidate_cached_user } = require("../../helpers"); +const { get_user, invalidate_cached_user } = require("../../helpers"); const { Context } = require("../../util/context"); const { asyncSafeSetInterval } = require("../../util/promise"); const { buffer_to_stream } = require("../../util/streamutil"); @@ -165,7 +165,8 @@ class DefaultUserService extends BaseService { ], ); user.password = password_hashed; - await generate_system_fsentries(user); + const svc_user = this.services.get('user'); + await svc_user.generate_default_fsentries({ user }); // generate default files for admin user const svc_fs = this.services.get('filesystem'); const make_tree_ = async ({ components, tree }) => { diff --git a/src/backend/src/routers/signup.js b/src/backend/src/routers/signup.js index 51644e5f..4d7d5f6e 100644 --- a/src/backend/src/routers/signup.js +++ b/src/backend/src/routers/signup.js @@ -345,7 +345,8 @@ module.exports = eggspress(['/signup'], { } } - await generate_system_fsentries(user); + const svc_user = Context.get('services').get('user'); + await svc_user.generate_default_fsentries({ user }); //set cookie res.cookie(config.cookie_name, token, { diff --git a/src/backend/src/services/UserService.js b/src/backend/src/services/UserService.js new file mode 100644 index 00000000..736d44a2 --- /dev/null +++ b/src/backend/src/services/UserService.js @@ -0,0 +1,111 @@ +const { LLMkdir } = require("../filesystem/ll_operations/ll_mkdir"); +const { RootNodeSelector } = require("../filesystem/node/selectors"); +const { invalidate_cached_user } = require("../helpers"); +const BaseService = require("./BaseService"); +const { DB_WRITE } = require("./database/consts"); + +class UserService extends BaseService { + static MODULES = { + uuidv4: require('uuid').v4, + }; + + async _init () { + this.db = this.services.get('database').get(DB_WRITE, 'user-service'); + } + + // used to be called: generate_system_fsentries + async generate_default_fsentries ({ user }) { + + this.log.noticeme('YES THIS WAS USED'); + + // Note: The comment below is outdated as we now do parallel writes for + // all filesystem operations. However, there may still be some + // performance hit so this requires further investigation. + + // Normally, it is recommended to use mkdir() to create new folders, + // but during signup this could result in multiple queries to the DB server + // and for servers in remote regions such as Asia this could result in a + // very long time for /signup to finish, sometimes up to 30-40 seconds! + // by combining as many queries as we can into one and avoiding multiple back-and-forth + // with the DB server, we can speed this process up significantly. + + const ts = Date.now()/1000; + + // Generate UUIDs for all the default folders and files + const uuidv4 = this.modules.uuidv4; + + let home_uuid = uuidv4(); + let trash_uuid = uuidv4(); + let appdata_uuid = uuidv4(); + let desktop_uuid = uuidv4(); + let documents_uuid = uuidv4(); + let pictures_uuid = uuidv4(); + let videos_uuid = uuidv4(); + let public_uuid = uuidv4(); + + const insert_res = await this.db.write( + `INSERT INTO fsentries + (uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable) VALUES + ( ?, ?, ?, ?, ?, true, ?, ?, true), + ( ?, ?, ?, ?, ?, true, ?, ?, true), + ( ?, ?, ?, ?, ?, true, ?, ?, true), + ( ?, ?, ?, ?, ?, true, ?, ?, true), + ( ?, ?, ?, ?, ?, true, ?, ?, true), + ( ?, ?, ?, ?, ?, true, ?, ?, true), + ( ?, ?, ?, ?, ?, true, ?, ?, true), + ( ?, ?, ?, ?, ?, true, ?, ?, true) + `, + [ + // Home + home_uuid, null, user.id, user.username, `/${user.username}`, ts, ts, + // Trash + trash_uuid, home_uuid, user.id, 'Trash', `/${user.username}/Trash`, ts, ts, + // AppData + appdata_uuid, home_uuid, user.id, 'AppData', `/${user.username}/AppData`, ts, ts, + // Desktop + desktop_uuid, home_uuid, user.id, 'Desktop', `/${user.username}/Desktop`, ts, ts, + // Documents + documents_uuid, home_uuid, user.id, 'Documents', `/${user.username}/Documents`, ts, ts, + // Pictures + pictures_uuid, home_uuid, user.id, 'Pictures', `/${user.username}/Pictures`, ts, ts, + // Videos + videos_uuid, home_uuid, user.id, 'Videos', `/${user.username}/Videos`, ts, ts, + // Public + public_uuid, home_uuid, user.id, 'Public', `/${user.username}/Public`, ts, ts, + ] + ); + + // https://stackoverflow.com/a/50103616 + let trash_id = insert_res.insertId; + let appdata_id = insert_res.insertId + 1; + let desktop_id = insert_res.insertId + 2; + let documents_id = insert_res.insertId + 3; + let pictures_id = insert_res.insertId + 4; + let videos_id = insert_res.insertId + 5; + let public_id = insert_res.insertId + 6; + + // Asynchronously set the user's system folders uuids in database + // This is for caching purposes, so we don't have to query the DB every time we need to access these folders + // This is also possible because we know the user's system folders uuids will never change + + // TODO: pass to IIAFE manager to avoid unhandled promise rejection + // (IIAFE manager doesn't exist yet, hence this is a TODO) + this.db.write( + `UPDATE user SET + trash_uuid=?, appdata_uuid=?, desktop_uuid=?, documents_uuid=?, pictures_uuid=?, videos_uuid=?, public_uuid=?, + trash_id=?, appdata_id=?, desktop_id=?, documents_id=?, pictures_id=?, videos_id=?, public_id=? + + WHERE id=?`, + [ + trash_uuid, appdata_uuid, desktop_uuid, documents_uuid, pictures_uuid, videos_uuid, public_uuid, + trash_id, appdata_id, desktop_id, documents_id, pictures_id, videos_id, public_id, + user.id + ] + ); + invalidate_cached_user(user); + } +} + +module.exports = { + UserService, +};