=f[n]&&n .
+ */
+"use strict"
+const express = require('express');
+const config = require('../config');
+const router = express.Router();
+const _path = require('path');
+const _fs = require('fs');
+const auth = require('../middleware/auth.js');
+const { generate_puter_page_html } = require('../temp/puter_page_loader');
+const { Context } = require('../util/context');
+const { DB_READ } = require('../services/database/consts');
+
+let auth_user;
+
+// -----------------------------------------------------------------------//
+// All other requests
+// -----------------------------------------------------------------------//
+router.all('*', async function(req, res, next) {
+ const subdomain = req.hostname.slice(0, -1 * (config.domain.length + 1));
+ let path = req.params[0] ? req.params[0] : 'index.html';
+
+ // --------------------------------------
+ // API
+ // --------------------------------------
+ if( subdomain === 'api'){
+ return next();
+ }
+ // --------------------------------------
+ // cloud.js must be accessible globally regardless of subdomain
+ // --------------------------------------
+ else if (path === '/cloud.js') {
+ return res.sendFile(_path.join(__dirname, '../../puter.js/alpha.js'), function (err) {
+ if (err && err.statusCode) {
+ return res.status(err.statusCode).send('Error /cloud.js')
+ }
+ });
+ }
+ // --------------------------------------
+ // /puter.js/v1 must be accessible globally regardless of subdomain
+ // --------------------------------------
+ else if (path === '/puter.js/v1' || path === '/puter.js/v1/') {
+ return res.sendFile(_path.join(__dirname, '../../puter.js/v1.js'), function (err) {
+ if (err && err.statusCode) {
+ return res.status(err.statusCode).send('Error /puter.js')
+ }
+ });
+ }
+ else if (path === '/puter.js/v2' || path === '/puter.js/v2/') {
+ return res.sendFile(_path.join(__dirname, '../../puter.js/v2.js'), function (err) {
+ if (err && err.statusCode) {
+ return res.status(err.statusCode).send('Error /puter.js')
+ }
+ });
+ }
+ // --------------------------------------
+ // https://js.[domain]/v1/
+ // --------------------------------------
+ else if( subdomain === 'js'){
+ if (path === '/v1' || path === '/v1/') {
+ return res.sendFile(_path.join(__dirname, '../../puter.js/v1.js'), function (err) {
+ if (err && err.statusCode) {
+ return res.status(err.statusCode).send('Error /puter.js')
+ }
+ });
+ }
+ if (path === '/v2' || path === '/v2/') {
+ return res.sendFile(_path.join(__dirname, '../../puter.js/v2.js'), function (err) {
+ if (err && err.statusCode) {
+ return res.status(err.statusCode).send('Error /puter.js')
+ }
+ });
+ }
+ }
+
+ const db = Context.get('services').get('database').get(DB_READ, 'default');
+
+ // --------------------------------------
+ // POST to login/signup/logout
+ // --------------------------------------
+ if( subdomain === '' && req.method === 'POST' &&
+ (
+ path === '/login' ||
+ path === '/signup' ||
+ path === '/logout' ||
+ path === '/send-pass-recovery-email' ||
+ path === '/set-pass-using-token'
+ )
+ ){
+ return next();
+ }
+ // --------------------------------------
+ // No subdomain: either GUI or landing pages
+ // --------------------------------------
+ else if( subdomain === ''){
+ // auth
+ const {jwt_auth, get_app, invalidate_cached_user} = require('../helpers');
+ let authed = false;
+ try{
+ try{
+ auth_user = await jwt_auth(req);
+ auth_user = auth_user.user;
+ authed = true;
+ }catch(e){
+ authed = false;
+ }
+ }
+ catch(e){
+ authed = false;
+ }
+
+ if(path === '/robots.txt'){
+ res.set('Content-Type', 'text/plain');
+ let r = ``;
+ r += `User-agent: AhrefsBot\nDisallow:/\n\n`;
+ r += `User-agent: BLEXBot\nDisallow: /\n\n`;
+ r += `User-agent: DotBot\nDisallow: /\n\n`;
+ r += `User-agent: ia_archiver\nDisallow: /\n\n`;
+ r += `User-agent: MJ12bot\nDisallow: /\n\n`;
+ r += `User-agent: SearchmetricsBot\nDisallow: /\n\n`;
+ r += `User-agent: SemrushBot\nDisallow: /\n\n`;
+ // sitemap
+ r += `\nSitemap: ${config.protocol}://${config.domain}/sitemap.xml\n`;
+ return res.send(r);
+ }
+ else if(path === '/sitemap.xml'){
+ let h = ``;
+ h += ``;
+ h += ``;
+
+ // docs
+ h += ``;
+ h += `${config.protocol}://docs.${config.domain}/ `;
+ h += ` `;
+
+ // apps
+ // TODO: use service for app discovery
+ let apps = await db.read( `SELECT * FROM apps WHERE approved_for_listing = 1`);
+ if(apps.length > 0){
+ for(let i=0; i`;
+ h += `${config.protocol}://${config.domain}/app/${app.name} `;
+ h += ``;
+ }
+ }
+ h += ` `;
+ res.set('Content-Type', 'application/xml');
+ return res.send(h);
+ }
+ else if(path === '/unsubscribe'){
+ let h = ``;
+ if(req.query.user_uuid === undefined)
+ h += 'user_uuid is required
';
+ else{
+ // modules
+ const {get_user} = require('../helpers')
+
+ // get user
+ const user = await get_user({uuid: req.query.user_uuid})
+
+ // more validation
+ if(user === undefined || user === null || user === false)
+ h += 'User not found.
';
+ else if(user.unsubscribed === 1)
+ h += 'You are already unsubscribed.
';
+ // mark user as confirmed
+ else{
+ await db.write(
+ "UPDATE `user` SET `unsubscribed` = 1 WHERE id = ?",
+ [user.id]
+ );
+
+ invalidate_cached_user(user);
+
+ // return results
+ h += `Your have successfully unsubscribed from all emails.
`;
+ }
+ }
+
+ h += ``;
+ res.send(h);
+ }
+ else if(path === '/confirm-email-by-token'){
+ let h = ``;
+ if(req.query.user_uuid === undefined)
+ h += 'user_uuid is required
';
+ else if(req.query.token === undefined)
+ h += 'token is required
';
+ else{
+ // modules
+ const {get_user} = require('../helpers')
+
+ // get user
+ const user = await get_user({uuid: req.query.user_uuid})
+
+ // more validation
+ if(user === undefined || user === null || user === false)
+ h += 'user not found.
';
+ else if(user.email_confirmed === 1)
+ h += 'Email already confirmed.
';
+ else if(user.email_confirm_token !== req.query.token)
+ h += 'invalid token.
';
+ // mark user as confirmed
+ else{
+ await db.write(
+ "UPDATE `user` SET `email_confirmed` = 1, `requires_email_confirmation` = 0 WHERE id = ?",
+ [user.id]
+ );
+ invalidate_cached_user(user);
+
+ // send realtime success msg to client
+ let socketio = require('../socketio.js').getio();
+ if(socketio){
+ socketio.to(user.id).emit('user.email_confirmed', {})
+ }
+
+ // return results
+ h += `Your email has been successfully confirmed.
`;
+
+ const svc_referralCode = Context.get('services').get('referral-code');
+ svc_referralCode.on_verified(user);
+ }
+ }
+
+ h += ``;
+ res.send(h);
+ }
+ // ------------------------
+ // /assets/
+ // ------------------------
+ else if (path.startsWith('/assets/')) {
+ return res.sendFile(path, { root: __dirname + '../../public' }, function (err) {
+ if (err && err.statusCode) {
+ return res.status(err.statusCode).send('Error /public/')
+ }
+ });
+ }
+ // ------------------------
+ // GUI
+ // ------------------------
+ else{
+ let canonical_url = config.origin + path;
+ let app_name, app_title, description;
+
+ // default title
+ app_title = config.title;
+
+ // /action/
+ if(path.startsWith('/action/')){
+ path = '/';
+ }
+ // /app/
+ else if(path.startsWith('/app/')){
+ app_name = path.replace('/app/', '');
+ const app = await get_app({name: app_name});
+ if(app){
+ app_title = app.title;
+ description = app.description;
+ }
+ // 404 - Not found!
+ else if(app_name){
+ app_title = app_name.charAt(0).toUpperCase() + app_name.slice(1);
+ res.status(404);
+ }
+
+ path = '/';
+ }
+
+ const manifest =
+ _fs.existsSync(_path.join(config.assets.gui, 'puter-gui.json'))
+ ? (() => {
+ const text = _fs.readFileSync(_path.join(config.assets.gui, 'puter-gui.json'), 'utf8');
+ return JSON.parse(text);
+ })()
+ : {};
+
+ // index.js
+ if(path === '/'){
+ const APP_ORIGIN = config.origin;
+ const API_ORIGIN = config.api_base_url;
+ return res.send(generate_puter_page_html({
+ app_origin: APP_ORIGIN,
+ api_origin: API_ORIGIN,
+
+ manifest,
+ gui_path: config.assets.gui,
+
+ // page meta
+ meta: {
+ title: app_title,
+ description: description || config.short_description,
+ short_description: config.short_description,
+ company: 'Puter Technologies Inc.',
+ canonical_url: canonical_url,
+ },
+
+ // gui parameters
+ gui_params: {
+ app_name_regex: config.app_name_regex,
+ app_name_max_length: config.app_name_max_length,
+ app_title_max_length: config.app_title_max_length,
+ subdomain_regex: config.subdomain_regex,
+ subdomain_max_length: config.subdomain_max_length,
+ domain: config.domain,
+ protocol: config.protocol,
+ env: config.env,
+ api_base_url: config.api_base_url,
+ thumb_width: config.thumb_width,
+ thumb_height: config.thumb_height,
+ contact_email: config.contact_email,
+ max_fsentry_name_length: config.max_fsentry_name_length,
+ require_email_verification_to_publish_website: config.require_email_verification_to_publish_website,
+ short_description: config.short_description,
+ long_description: config.long_description,
+ },
+ }));
+ }
+
+ // /dist/...
+ else if(path.startsWith('/dist/') || path.startsWith('/src/')){
+ path = _path.resolve(path);
+ return res.sendFile(path, {root: config.assets.gui}, function(err){
+ if(err && err.statusCode){
+ return res.status(err.statusCode).send('Error /gui/dist/')
+ }
+ });
+ }
+
+ // All other paths
+ else{
+ return res.sendFile(path, {root: _path.join(config.assets.gui, 'src')}, function(err){
+ if(err && err.statusCode){
+ return res.status(err.statusCode).send('Error /gui/')
+ }
+ });
+ }
+ }
+ }
+ // --------------------------------------
+ // Native Apps
+ // --------------------------------------
+ else if(subdomain === 'viewer' || subdomain === 'editor' || subdomain === 'about' || subdomain === 'docs' ||
+ subdomain === 'player' || subdomain === 'pdf' || subdomain === 'code' || subdomain === 'markus' ||
+ subdomain === 'draw' || subdomain === 'camera' || subdomain === 'recorder' ||
+ subdomain === 'dev-center' || subdomain === 'terminal'){
+
+ let root = _path.join(__dirname, '../../apps/', subdomain);
+ if ( subdomain === 'docs' ) root += '/dist';
+ root = _path.normalize(root);
+
+ path = _path.normalize(path);
+ const real_path = _path.normalize(_path.join(root, path));
+
+ // Determine if the path is a directory
+ // (necessary because otherwise res.sendFile() will HANG!)
+ try {
+ const is_dir = (await _fs.promises.stat(real_path)).isDirectory();
+ if ( is_dir && ! path.endsWith('/') ) {
+ // Redirect to directory (use 307 to avoid browser caching)
+ path += '/';
+ let redirect_url = req.protocol + '://' + req.get('host') + path;
+
+ // We need to add the query string to the redirect URL
+ if ( req.query ) {
+ const old_url = req.protocol + '://' + req.get('host') + req.originalUrl;
+ redirect_url += new URL(old_url).search;
+ }
+
+ return res.redirect(307, redirect_url);
+ }
+ } catch (e) {
+ console.error(e);
+ return res.status(404).send('Not found');
+ }
+
+ console.log('sending path', path, 'from', root);
+ try {
+ return res.sendFile(path, { root }, function(err){
+ if(err && err.statusCode){
+ return res.status(err.statusCode).send('Error /apps/')
+ }
+ });
+ } catch (e) {
+ console.error('error from sendFile', e);
+ return res.status(err.statusCode).send('Error /apps/')
+ }
+ }
+ // --------------------------------------
+ // WWW, redirect to root domain
+ // --------------------------------------
+ else if( subdomain === 'www'){
+ console.log('redirecting from www to root domain');
+ return res.redirect(config.origin);
+ }
+ //------------------------------------------
+ // User-defined subdomains: *.puter.com
+ // redirect to static hosting domain *.puter.site
+ //------------------------------------------
+ else{
+ // replace hostname with static hosting domain and redirect to the same path
+ return res.redirect(301, req.protocol + '://' + req.get('host').replace(config.domain, config.static_hosting_domain) + req.originalUrl);
+ }
+});
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/apps.js b/packages/backend/src/routers/apps.js
new file mode 100644
index 00000000..dbc470ec
--- /dev/null
+++ b/packages/backend/src/routers/apps.js
@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+const { app_name_exists, refresh_apps_cache, chkperm, convert_path_to_fsentry, get_app } = require('../helpers');
+const { DB_WRITE, DB_READ } = require('../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// GET /apps
+// -----------------------------------------------------------------------//
+router.get('/apps', auth, express.json({limit: '50mb'}), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ const db = req.services.get('database').get(DB_READ, 'apps');
+
+ let apps_res = await dbrr.read(
+ `SELECT * FROM apps WHERE owner_user_id = ? ORDER BY timestamp DESC`,
+ [req.user.id]
+ );
+
+ const svc_appInformation = req.services.get('app-information');
+
+ let apps=[];
+
+ if(apps_res.length > 0){
+ for(let i=0; i< apps_res.length; i++){
+ // filetype associations
+ let ftassocs = await db.read(
+ `SELECT * FROM app_filetype_association WHERE app_id = ?`,
+ [apps_res[i].id]
+ );
+
+ let filetype_associations = []
+ if(ftassocs.length > 0){
+ ftassocs.forEach(ftassoc => {
+ filetype_associations.push(ftassoc.type);
+ });
+ }
+
+ const stats = await svc_appInformation.get_stats(apps_res[i].uid);
+
+ apps.push({
+ uid: apps_res[i].uid,
+ name: apps_res[i].name,
+ description: apps_res[i].description,
+ title: apps_res[i].title,
+ icon: apps_res[i].icon,
+ index_url: apps_res[i].index_url,
+ godmode: apps_res[i].godmode,
+ maximize_on_start: apps_res[i].maximize_on_start,
+ filetype_associations: filetype_associations,
+ ...stats,
+ approved_for_incentive_program: apps_res[i].approved_for_incentive_program,
+ created_at: apps_res[i].timestamp,
+ })
+ }
+ }
+
+ return res.send(apps);
+})
+
+// -----------------------------------------------------------------------//
+// GET /apps/:name(s)
+// -----------------------------------------------------------------------//
+router.get('/apps/:name', auth, express.json({limit: '50mb'}), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ let app_names = req.params.name.split('|');
+ let retobj = [];
+
+ if(app_names.length > 0){
+ // prepare each app for returning to user
+ for (let index = 0; index < app_names.length; index++) {
+ const app = await get_app({name: app_names[index]});
+ let final_obj = {};
+ if(app){
+ final_obj = {
+ uuid: app.uid,
+ name: app.name,
+ title: app.title,
+ icon: app.icon,
+ godmode: app.godmode,
+ maximize_on_start: app.maximize_on_start,
+ index_url: app.index_url,
+ };
+ }
+ // add to object to be returned
+ retobj.push(final_obj)
+ }
+ }
+
+ // order output based on input!
+ let final_obj = [];
+ for (let index = 0; index < app_names.length; index++) {
+ const app_name = app_names[index];
+ for (let index = 0; index < retobj.length; index++) {
+ if(retobj[index].name === app_name)
+ final_obj.push(retobj[index]);
+ }
+ }
+
+ return res.send(final_obj);
+})
+
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/auth/app-uid-from-origin.js b/packages/backend/src/routers/auth/app-uid-from-origin.js
new file mode 100644
index 00000000..fba2eafa
--- /dev/null
+++ b/packages/backend/src/routers/auth/app-uid-from-origin.js
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../api/APIError");
+const eggspress = require("../../api/eggspress");
+const { UserActorType } = require("../../services/auth/Actor");
+const { Context } = require("../../util/context");
+
+module.exports = eggspress('/auth/app-uid-from-origin', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['POST', 'GET'],
+}, async (req, res, next) => {
+ const x = Context.get();
+ const svc_auth = x.get('services').get('auth');
+
+ const origin = req.body.origin || req.query.origin;
+
+ if ( ! origin ) {
+ throw APIError.create('field_missing', null, { key: 'origin' });
+ }
+
+ res.json({
+ uid: await svc_auth.app_uid_from_origin(origin),
+ });
+});
diff --git a/packages/backend/src/routers/auth/check-app.js b/packages/backend/src/routers/auth/check-app.js
new file mode 100644
index 00000000..91f22240
--- /dev/null
+++ b/packages/backend/src/routers/auth/check-app.js
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../api/APIError");
+const eggspress = require("../../api/eggspress");
+const { LLMkdir } = require("../../filesystem/ll_operations/ll_mkdir");
+const { NodeUIDSelector, NodePathSelector } = require("../../filesystem/node/selectors");
+const { NodeChildSelector } = require("../../filesystem/node/selectors");
+const { get_app } = require("../../helpers");
+const { UserActorType, Actor, AppUnderUserActorType } = require("../../services/auth/Actor");
+const { Context } = require("../../util/context");
+
+module.exports = eggspress('/auth/check-app', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const x = Context.get();
+ const svc_auth = x.get('services').get('auth');
+ const svc_permission = x.get('services').get('permission');
+
+ // Only users can get user-app tokens
+ const actor = Context.get('actor');
+ if ( ! (actor.type instanceof UserActorType) ) {
+ throw APIError.create('forbidden');
+ }
+
+ if ( req.body.app_uid === undefined && req.body.origin === undefined ) {
+ throw APIError.create('field_missing', null, {
+ // TODO: standardize a way to provide multiple options
+ key: 'app_uid or origin',
+ });
+ }
+
+ const app_uid = req.body.app_uid ??
+ await svc_auth.app_uid_from_origin(req.body.origin);
+
+ const app = await get_app({ uid: app_uid });
+ if ( ! app ) {
+ throw APIError.create('app_does_not_exist', null, {
+ identifier: app_uid,
+ });
+ }
+
+ const user = actor.type.user;
+
+ const app_actor = new Actor({
+ user_uid: user.uuid,
+ app_uid,
+ type: new AppUnderUserActorType({
+ user,
+ app,
+ }),
+ });
+
+ const authenticated = !! await svc_permission.check(app_actor, 'flag:app-is-authenticated');
+
+ let token;
+ if ( authenticated ) token = await svc_auth.get_user_app_token(app_uid);
+
+ res.json({
+ ...(token ? { token } : {}),
+ app_uid: app_uid ||
+ await svc_auth.app_uid_from_origin(req.body.origin),
+ authenticated,
+ });
+});
+
diff --git a/packages/backend/src/routers/auth/create-access-token.js b/packages/backend/src/routers/auth/create-access-token.js
new file mode 100644
index 00000000..6d285764
--- /dev/null
+++ b/packages/backend/src/routers/auth/create-access-token.js
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../api/APIError");
+const eggspress = require("../../api/eggspress");
+const { Context } = require("../../util/context");
+
+module.exports = eggspress('/auth/create-access-token', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const x = Context.get();
+ const svc_auth = x.get('services').get('auth');
+
+ const permissions = req.body.permissions || [];
+
+ if ( permissions.length === 0 ) {
+ throw APIError.create('field_missing', null, { key: 'permissions' });
+ }
+
+ for ( let i=0 ; i < permissions.length ; i++ ) {
+ let perm = permissions[i];
+ if ( typeof perm === 'string' ) {
+ perm = permissions[i] = [perm];
+ }
+ if ( ! Array.isArray(perm) ) {
+ throw APIError.create('field_invalid', null, { key: 'permissions' });
+ }
+ if ( perm.length === 0 || perm.length > 2 ) {
+ throw APIError.create('field_invalid', null, { key: 'permissions' });
+ }
+ if ( typeof perm[0] !== 'string' ) {
+ throw APIError.create('field_invalid', null, { key: 'permissions' });
+ }
+ if ( perm.length === 2 && typeof perm[1] !== 'object' ) {
+ throw APIError.create('field_invalid', null, { key: 'permissions' });
+ }
+ }
+
+ const actor = Context.get('actor');
+
+ const token = await svc_auth.create_access_token(
+ actor, permissions
+ );
+
+ res.json({ token });
+});
diff --git a/packages/backend/src/routers/auth/get-user-app-token.js b/packages/backend/src/routers/auth/get-user-app-token.js
new file mode 100644
index 00000000..b7c02b1a
--- /dev/null
+++ b/packages/backend/src/routers/auth/get-user-app-token.js
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../api/APIError");
+const eggspress = require("../../api/eggspress");
+const { LLMkdir } = require("../../filesystem/ll_operations/ll_mkdir");
+const { NodeUIDSelector, NodePathSelector } = require("../../filesystem/node/selectors");
+const { NodeChildSelector } = require("../../filesystem/node/selectors");
+const { get_app } = require("../../helpers");
+const { UserActorType } = require("../../services/auth/Actor");
+const { Context } = require("../../util/context");
+
+module.exports = eggspress('/auth/get-user-app-token', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const x = Context.get();
+ const svc_auth = x.get('services').get('auth');
+
+ // Only users can get user-app tokens
+ const actor = Context.get('actor');
+ if ( ! (actor.type instanceof UserActorType) ) {
+ throw APIError.create('forbidden');
+ }
+
+ if ( req.body.app_uid === undefined && req.body.origin === undefined ) {
+ throw APIError.create('field_missing', null, {
+ // TODO: standardize a way to provide multiple options
+ key: 'app_uid or origin',
+ });
+ }
+
+ const token = ( req.body.app_uid !== undefined )
+ ? await svc_auth.get_user_app_token(req.body.app_uid)
+ : await svc_auth.get_user_app_token_from_origin(req.body.origin)
+ ;
+
+ const app_uid = req.body.app_uid ??
+ await svc_auth.app_uid_from_origin(req.body.origin);
+
+ const app = await get_app({ uid: app_uid });
+ if ( ! app ) {
+ throw APIError.create('app_does_not_exist', null, {
+ identifier: app_uid,
+ });
+ }
+
+ const svc_fs = x.get('services').get('filesystem');
+ const appdata_dir_sel = actor.type.user.appdata_uuid
+ ? new NodeUIDSelector(actor.type.user.appdata_uuid)
+ : new NodePathSelector(`/${actor.type.user.username}/AppData`);
+ const appdata_app_dir_node = await svc_fs.node(new NodeChildSelector(
+ appdata_dir_sel,
+ app_uid,
+ ));
+
+ if ( ! await appdata_app_dir_node.exists() ) {
+ const ll_mkdir = new LLMkdir();
+ await ll_mkdir.run({
+ thumbnail: app.icon,
+ parent: await svc_fs.node(appdata_dir_sel),
+ name: app_uid,
+ user: actor.type.user,
+ });
+ }
+
+ const svc_permission = x.get('services').get('permission');
+ svc_permission.grant_user_app_permission(actor, app_uid, 'flag:app-is-authenticated');
+
+ res.json({
+ token,
+ app_uid: app_uid ||
+ await svc_auth.app_uid_from_origin(req.body.origin),
+ });
+});
diff --git a/packages/backend/src/routers/auth/grant-user-app.js b/packages/backend/src/routers/auth/grant-user-app.js
new file mode 100644
index 00000000..3fe570fa
--- /dev/null
+++ b/packages/backend/src/routers/auth/grant-user-app.js
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../api/APIError");
+const eggspress = require("../../api/eggspress");
+const { UserActorType } = require("../../services/auth/Actor");
+const { Context } = require("../../util/context");
+
+module.exports = eggspress('/auth/grant-user-app', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const x = Context.get();
+ const svc_permission = x.get('services').get('permission');
+
+ // Only users can grant user-app permissions
+ const actor = Context.get('actor');
+ if ( ! (actor.type instanceof UserActorType) ) {
+ throw APIError.create('forbidden');
+ }
+
+ if ( req.body.origin ) {
+ const svc_auth = x.get('services').get('auth');
+ req.body.app_uid = await svc_auth.app_uid_from_origin(req.body.origin);
+ }
+
+ if ( ! req.body.app_uid ) {
+ throw APIError.create('field_missing', null, { key: 'app_uid' });
+ }
+
+ const token = await svc_permission.grant_user_app_permission(
+ actor, req.body.app_uid, req.body.permission,
+ req.body.extra || {}, req.body.meta || {}
+ );
+
+ res.json({});
+});
+
diff --git a/packages/backend/src/routers/auth/list-permissions.js b/packages/backend/src/routers/auth/list-permissions.js
new file mode 100644
index 00000000..e1571af3
--- /dev/null
+++ b/packages/backend/src/routers/auth/list-permissions.js
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const eggspress = require("../../api/eggspress");
+const { get_app } = require("../../helpers");
+const { UserActorType } = require("../../services/auth/Actor");
+const { DB_READ } = require("../../services/database/consts");
+const { Context } = require("../../util/context");
+
+module.exports = eggspress('/auth/list-permissions', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['GET'],
+}, async (req, res, next) => {
+ const x = Context.get();
+
+ const actor = x.get('actor');
+
+ // Apps cannot (currently) check permissions on behalf of users
+ if ( ! ( actor.type instanceof UserActorType ) ) {
+ throw APIError.create('forbidden');
+ }
+
+ const db = x.get('services').get('database').get(DB_READ, 'permissions');
+
+ const permissions = [];
+
+ const rows = await db.read(
+ 'SELECT * FROM `user_to_app_permissions` WHERE user_id=?',
+ [ actor.type.user.id ]
+ );
+
+ for ( const row of rows ) {
+ const app = await get_app({ id: row.app_id });
+
+ delete app.id;
+ delete app.approved_for_listing;
+ delete app.approved_for_opening_items;
+ delete app.godmode;
+ delete app.owner_user_id;
+
+ const permission = {
+ app,
+ permission: row.permission,
+ extra: row.extra
+ };
+
+ permissions.push(permission);
+ }
+
+ res.json(permissions);
+});
diff --git a/packages/backend/src/routers/auth/revoke-user-app.js b/packages/backend/src/routers/auth/revoke-user-app.js
new file mode 100644
index 00000000..00d81b38
--- /dev/null
+++ b/packages/backend/src/routers/auth/revoke-user-app.js
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const eggspress = require("../../api/eggspress");
+const { UserActorType } = require("../../services/auth/Actor");
+const { Context } = require("../../util/context");
+
+module.exports = eggspress('/auth/revoke-user-app', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const x = Context.get();
+ const svc_permission = x.get('services').get('permission');
+
+ // Only users can grant user-app permissions
+ const actor = Context.get('actor');
+ if ( ! (actor.type instanceof UserActorType) ) {
+ throw APIError.create('forbidden');
+ }
+
+ if ( req.body.origin ) {
+ const svc_auth = x.get('services').get('auth');
+ req.body.app_uid = await svc_auth.app_uid_from_origin(req.body.origin);
+ }
+
+ if ( ! req.body.app_uid ) {
+ throw APIError.create('field_missing', null, { key: 'app_uid' });
+ }
+
+ if ( req.body.permission === '*' ) {
+ await svc_permission.revoke_user_app_all(
+ actor, req.body.app_uid, req.body.meta || {},
+ );
+ }
+
+ const token = await svc_permission.revoke_user_app_permission(
+ actor, req.body.app_uid, req.body.permission,
+ req.body.meta || {},
+ );
+
+ res.json({});
+});
+
+
diff --git a/packages/backend/src/routers/change_email.js b/packages/backend/src/routers/change_email.js
new file mode 100644
index 00000000..91bb9b23
--- /dev/null
+++ b/packages/backend/src/routers/change_email.js
@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const validator = require('validator');
+const crypto = require('crypto');
+const eggspress = require('../api/eggspress.js');
+const APIError = require('../api/APIError.js');
+const { DB_READ, DB_WRITE } = require('../services/database/consts.js');
+
+const CHANGE_EMAIL_START = eggspress('/change_email/start', {
+ subdomain: 'api',
+ auth: true,
+ verified: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const user = req.user;
+ const new_email = req.body.new_email;
+
+ // TODO: DRY: signup.js
+ // validation
+ if( ! new_email ) {
+ throw APIError.create('field_missing', null, { key: 'new_email' });
+ }
+ if ( typeof new_email !== 'string' ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'new_email', expected: 'a valid email address' });
+ }
+ if ( ! validator.isEmail(new_email) ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'new_email', expected: 'a valid email address' });
+ }
+
+ // check if email is already in use
+ const svc_mysql = req.services.get('mysql');
+ const db = req.services.get('database').get(DB_WRITE, 'auth');
+ const rows = await db.read(
+ 'SELECT COUNT(*) AS `count` FROM `user` WHERE `email` = ?',
+ [new_email]
+ );
+ if ( rows[0].count > 0 ) {
+ throw APIError.create('email_already_in_use', null, { email: new_email });
+ }
+
+ // generate confirmation token
+ const token = crypto.randomBytes(4).toString('hex');
+
+ // update user
+ await db.write(
+ 'UPDATE `user` SET `unconfirmed_change_email` = ?, `change_email_confirm_token` = ? WHERE `id` = ?',
+ [new_email, token, user.id]
+ );
+});
+
+const CHANGE_EMAIL_CONFIRM = eggspress('/change_email/confirm', {
+ subdomain: 'api',
+ auth: true,
+ verified: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const user = req.user;
+ const token = req.body.token;
+
+ if ( ! token ) {
+ throw APIError.create('field_missing', null, { key: 'token' });
+ }
+
+ const svc_mysql = req.services.get('mysql');
+ const dbrr = svc_mysql.get(DB_MODE_READ, 'change-email-confirm');
+ const [rows] = await dbrr.promise().query(
+ 'SELECT `unconfirmed_change_email` FROM `user` WHERE `id` = ? AND `change_email_confirm_token` = ?',
+ [user.id, token]
+ );
+ if ( rows.length === 0 ) {
+ throw APIError.create('token_invalid');
+ }
+
+ const new_email = rows[0].unconfirmed_change_email;
+
+ const db = req.services.get('database').get(DB_WRITE, 'auth');
+ await db.write(
+ 'UPDATE `user` SET `email` = ?, `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL WHERE `id` = ?',
+ [new_email, user.id]
+ );
+});
+
+module.exports = app => {
+ app.use(CHANGE_EMAIL_START);
+ app.use(CHANGE_EMAIL_CONFIRM);
+}
diff --git a/packages/backend/src/routers/change_username.js b/packages/backend/src/routers/change_username.js
new file mode 100644
index 00000000..d03e1225
--- /dev/null
+++ b/packages/backend/src/routers/change_username.js
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const config = require('../config');
+const eggspress = require('../api/eggspress.js');
+const { Context } = require('../util/context.js');
+const { UserActorType } = require('../services/auth/Actor.js');
+const APIError = require('../api/APIError.js');
+const { DB_WRITE } = require('../services/database/consts');
+
+module.exports = eggspress('/change_username', {
+ subdomain: 'api',
+ auth2: true,
+ verified: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+
+ const {username_exists, change_username} = require('../helpers');
+
+ const actor = Context.get('actor');
+
+ // Only users can change their username (apps can't do this)
+ if ( ! ( actor.type instanceof UserActorType ) ) {
+ throw APIError.create('forbidden');
+ }
+
+ // validation
+ if(!req.body.new_username)
+ throw APIError.create('field_missing', null, { key: 'new_username' });
+ // new_username must be a string
+ else if(typeof req.body.new_username !== 'string')
+ throw APIError.create('field_invalid', null, { key: 'new_username', expected: 'a string' });
+ else if(!req.body.new_username.match(config.username_regex))
+ throw APIError.create('field_invalid', null, { key: 'new_username', expected: 'letters, numbers, underscore (_)' });
+ else if(req.body.new_username.length > config.username_max_length)
+ throw APIError.create('field_too_long', null, { key: 'new_username', max_length: config.username_max_length });
+ // duplicate username check
+ if(await username_exists(req.body.new_username))
+ throw APIError.create('username_already_in_use', null, { username: req.body.new_username });
+
+ const db = Context.get('services').get('database').get(DB_WRITE, 'auth');
+
+ // Has the user already changed their username twice this month?
+ const rows = await db.read(
+ 'SELECT COUNT(*) AS `count` FROM `user_update_audit` ' +
+ 'WHERE `user_id`=? AND `reason`=? AND `created_at` > DATE_SUB(NOW(), INTERVAL 1 MONTH)',
+ [ req.user.id, 'change_username' ]
+ );
+
+ if ( rows[0].count >= 2 ) {
+ throw APIError.create('too_many_username_changes');
+ }
+
+ // Update username change audit table
+ await db.write(
+ 'INSERT INTO `user_update_audit` ' +
+ '(`user_id`, `user_id_keep`, `old_username`, `new_username`, `reason`) ' +
+ 'VALUES (?, ?, ?, ?, ?)',
+ [
+ req.user.id, req.user.id,
+ req.user.username, req.body.new_username,
+ 'change_username'
+ ]
+ );
+
+ await change_username(req.user.id, req.body.new_username)
+
+ res.json({});
+});
diff --git a/packages/backend/src/routers/confirm-email.js b/packages/backend/src/routers/confirm-email.js
new file mode 100644
index 00000000..39fda069
--- /dev/null
+++ b/packages/backend/src/routers/confirm-email.js
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const { invalidate_cached_user } = require('../helpers');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+const { DB_WRITE } = require('../services/database/consts');
+
+// -----------------------------------------------------------------------//
+// POST /confirm-email
+// -----------------------------------------------------------------------//
+router.post('/confirm-email', auth, express.json(), async (req, res, next)=>{
+ // Either api. subdomain or no subdomain
+ if(require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '')
+ next();
+
+ if(!req.body.code)
+ req.status(400).send('code is required');
+
+ // Modules
+ const db = req.services.get('database').get(DB_WRITE, 'auth');
+
+ // Increment & check rate limit
+ if(kv.incr(`confirm-email|${req.ip}|${req.body.email ?? req.body.username}`) > 10)
+ return res.status(429).send({error: 'Too many requests.'});
+ // Set expiry for rate limit
+ kv.expire(`confirm-email|${req.ip}|${req.body.email ?? req.body.username}`, 60 * 10, 'NX')
+
+ if(req.body.code === req.user.email_confirm_code) {
+ await db.write(
+ "UPDATE `user` SET `email_confirmed` = 1, `requires_email_confirmation` = 0 WHERE id = ? LIMIT 1",
+ [req.user.id],
+ );
+ invalidate_cached_user(req.user);
+ }
+
+ // Build response object
+ const res_obj = {
+ email_confirmed: (req.body.code === req.user.email_confirm_code),
+ original_client_socket_id: req.body.original_client_socket_id,
+ }
+
+ // Send realtime success msg to client
+ if(req.body.code === req.user.email_confirm_code){
+ let socketio = require('../socketio.js').getio();
+ if(socketio){
+ socketio.to(req.user.id).emit('user.email_confirmed', {original_client_socket_id: req.body.original_client_socket_id})
+ }
+ }
+
+ // return results
+ return res.send(res_obj)
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/contactUs.js b/packages/backend/src/routers/contactUs.js
new file mode 100644
index 00000000..2ac09f32
--- /dev/null
+++ b/packages/backend/src/routers/contactUs.js
@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+const { get_user, generate_random_str } = require('../helpers');
+const { DB_WRITE } = require('../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /contactUs
+// -----------------------------------------------------------------------//
+router.post('/contactUs', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // message is required
+ if(!req.body.message)
+ return res.status(400).send({message: 'message is required'})
+ // message must be a string
+ if(typeof req.body.message !== 'string')
+ return res.status(400).send('message must be a string.')
+ // message is too long
+ else if(req.body.message.length > 100000)
+ return res.status(400).send({message: 'message is too long'})
+
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'feedback');
+
+ try{
+ db.write(
+ `INSERT INTO feedback
+ (user_id, message) VALUES
+ ( ?, ?)`,
+ [
+ //user_id
+ req.user.id,
+ //message
+ req.body.message,
+ ]
+ );
+
+ // get user
+ let user = await get_user({id: req.user.id});
+
+ // send email to support
+ const nodemailer = require("nodemailer");
+
+ // send email notif
+ let transporter = nodemailer.createTransport({
+ host: config.smtp_server,
+ port: config.smpt_port,
+ secure: true, // STARTTLS
+ auth: {
+ user: config.smtp_username,
+ pass: config.smtp_password,
+ },
+ });
+
+ transporter.sendMail({
+ from: '"Puter" no-reply@puter.com', // sender address
+ to: 'support@puter.com', // list of receivers
+ replyTo: user.email === null ? undefined : user.email,
+ subject: `Your Feedback/Support Request (#${generate_random_str(4)})`, // Subject line
+ text: req.body.message,
+ });
+
+ return res.send({});
+ }catch(e){
+ return res.status(400).send(e);
+ }
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/delete-site.js b/packages/backend/src/routers/delete-site.js
new file mode 100644
index 00000000..7137d48a
--- /dev/null
+++ b/packages/backend/src/routers/delete-site.js
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+
+// -----------------------------------------------------------------------//
+// POST /delete-site
+// -----------------------------------------------------------------------//
+router.post('/delete-site', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // validation
+ if(req.body.site_uuid === undefined)
+ return res.status(400).send('site_uuid is required')
+
+ // modules
+ const {} = require('../helpers');
+ const db = require('../db/mysql.js')
+
+ await db.promise().execute(
+ `DELETE FROM subdomains WHERE user_id = ? AND uuid = ?`,
+ [req.user.id, req.body.site_uuid]
+ );
+ res.send({});
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/df.js b/packages/backend/src/routers/df.js
new file mode 100644
index 00000000..1c8ddc50
--- /dev/null
+++ b/packages/backend/src/routers/df.js
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const config = require('../config.js');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+
+// -----------------------------------------------------------------------//
+// POST /df
+// -----------------------------------------------------------------------//
+router.post('/df', auth, express.json(), async (req, response, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return response.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ const {df} = require('../helpers');
+ try{
+ // auth
+ response.send({
+ used: parseInt(await df(req.user.id)),
+ capacity: (req.user.free_storage === undefined || req.user.free_storage === null) ? config.storage_capacity : req.user.free_storage,
+ });
+ }catch(e){
+ console.log(e)
+ response.status(400).send()
+ }
+})
+
+// -----------------------------------------------------------------------//
+// GET /df
+// -----------------------------------------------------------------------//
+router.get('/df', auth, express.json(), async (req, response, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return response.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ const {df} = require('../helpers');
+ try{
+ // auth
+ response.send({
+ used: parseInt(await df(req.user.id)),
+ capacity: (req.user.free_storage === undefined || req.user.free_storage === null) ? config.storage_capacity : req.user.free_storage,
+ });
+ }catch(e){
+ console.log(e)
+ response.status(400).send()
+ }
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/download.js b/packages/backend/src/routers/download.js
new file mode 100644
index 00000000..6ce38c2e
--- /dev/null
+++ b/packages/backend/src/routers/download.js
@@ -0,0 +1,273 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const config = require('../config');
+const axios = require('axios');
+const mime = require('mime-types')
+const path = require('path')
+const https = require('https');
+const {URL} = require('url');
+const _path = require('path');
+const { NodePathSelector } = require('../filesystem/node/selectors.js');
+const eggspress = require('../api/eggspress.js');
+const { Context } = require('../util/context');
+const { HLWrite } = require('../filesystem/hl_operations/hl_write');
+const FSNodeParam = require('../api/filesystem/FSNodeParam');
+// TODO: eggspressify
+
+// todo this could be abused to send get requests to any url and cause a denial of service
+//
+// -----------------------------------------------------------------------//
+// POST /download
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/download', {
+ subdomain: 'api',
+ verified: true,
+ auth: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['POST'],
+ parameters: {
+ fsNode: new FSNodeParam('path'),
+ }
+}, async (req, res, next) => {
+ const log = req.services.get('log-service').create('api:download');
+
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // socketio
+ let socketio = require('../socketio.js').getio();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // modules
+ const {cp, id2path, suggest_app_for_fsentry, validate_signature_auth, is_shared_with_anyone, uuid2fsentry} = require('../helpers')
+ if(!req.body.url)
+ return res.status(400).send({code: 'url_is_required', message: 'URL is required'});
+
+ // if url doesn't have a protocol, add http://
+ if(!req.body.url.startsWith('http://') && !req.body.url.startsWith('https://')){
+ req.body.url = 'http://' + req.body.url;
+ }
+
+ // Ensure url is not "localhost" or a private IP range
+ {
+ const url_obj = new URL(req.body.url);
+
+ if ( url_obj.hostname === 'localhost' ) {
+ return res.status(400).send({code: 'invalid_url', message: 'Invalid URL'});
+ }
+
+ // GitHub Copilot generated most of these
+ if ( url_obj.hostname.match(/^10\./) ) {
+ return res.status(400).send({code: 'invalid_url', message: 'Invalid URL'});
+ }
+ if ( url_obj.hostname.match(/^192\.168\./) ) {
+ return res.status(400).send({code: 'invalid_url', message: 'Invalid URL'});
+ }
+ if ( url_obj.hostname.match(/^172\.(1[6-9]|2[0-9]|3[0-1])\./) ) {
+ return res.status(400).send({code: 'invalid_url', message: 'Invalid URL'});
+ }
+ // github 127
+ if ( url_obj.hostname.match(/^127\./) ) {
+ return res.status(400).send({code: 'invalid_url', message: 'Invalid URL'});
+ }
+ // and 100 range for tailscale
+ if ( url_obj.hostname.match(/^100\./) ) {
+ return res.status(400).send({code: 'invalid_url', message: 'Invalid URL'});
+ }
+ }
+
+
+ // check if `url` is a valid URL and then parse it
+ let url_obj;
+ try{
+ url_obj = new URL(req.body.url);
+ }catch(e){
+ return res.status(400).send({code: 'invalid_url', message: 'Invalid URL'});
+ }
+
+ //--------------------------------------------------
+ // Puter file
+ //--------------------------------------------------
+ if(url_obj.origin === config.api_base_url && url_obj.searchParams && url_obj.searchParams.get('uid') && url_obj.searchParams.get('expires') && url_obj.searchParams.get('signature')){
+ // authenticate
+ // validate URL signature
+ try{
+ validate_signature_auth(req.body.url, 'read');
+ }
+ catch(e){
+ console.log(e)
+ return res.status(403).send(e);
+ }
+
+ // get file
+ const source_item = await uuid2fsentry(url_obj.searchParams.get('uid'));
+ log.info('source_item', { value: source_item });
+ const source_path = await id2path(source_item.id);
+ let new_fsentries
+ try{
+ new_fsentries = await cp(source_path, req.body.path, req.user, false, true, false);
+ }catch(e){
+ return res.status(400).send(e)
+ }
+ // is_shared, dirpath, original_client_socket_id, suggested_apps,...
+ if(new_fsentries.length > 0){
+ for (let i = 0; i < new_fsentries.length; i++) {
+ let fse = await uuid2fsentry(new_fsentries[i].uid);
+ new_fsentries[i].is_shared = await is_shared_with_anyone(fse.id);
+ new_fsentries[i].dirpath = _path.dirname(new_fsentries[i].path);
+ new_fsentries[i].original_client_socket_id = req.body.original_client_socket_id;
+ new_fsentries[i].suggested_apps = await suggest_app_for_fsentry(fse, {user: req.user});
+
+ // associated_app
+ if(new_fsentries[i].associated_app_id){
+ const app = await get_app({id: new_fsentries[i].associated_app_id})
+ // remove some privileged information
+ delete app.id;
+ delete app.approved_for_listing;
+ delete app.approved_for_opening_items;
+ delete app.godmode;
+ delete app.owner_user_id;
+ // add to array
+ new_fsentries[i].associated_app = app;
+ }else{
+ new_fsentries[i].associated_app = {};
+ }
+
+ // send realtime msg to client
+ if(socketio){
+ socketio.to(req.user.id).emit('item.added', new_fsentries[i])
+ }
+ }
+ }
+
+ return res.status(200).send(new_fsentries);
+ }
+
+ //--------------------------------------------------
+ // non-Puter file
+ //--------------------------------------------------
+
+ // disable axios ssl verification when in dev env and the url is from the local Puter API
+ let axios_instance;
+ if(config.env === 'dev' && url_obj.origin === config.api_base_url){
+ axios_instance = axios.create({
+ httpsAgent: new https.Agent({
+ rejectUnauthorized: false
+ })
+ });
+ }else{
+ axios_instance = axios;
+ }
+
+ // todo if there is a way to get the file size without downloading the whole file, do that and then check if user has enough space
+
+ // get file
+ let file;
+ try{
+ // old implementation using buffer
+ file = await axios_instance.get(req.body.url, {responseType: 'arraybuffer', onDownloadProgress: (progressEvent) => {
+ if(req.body.socket_id){
+ socketio.to(req.body.socket_id).emit('download.progress', {
+ loaded: progressEvent.loaded,
+ total: progressEvent.total * 2,
+ operation_id: req.body.operation_id,
+ item_upload_id: req.body.item_upload_id,
+ original_client_socket_id: req.body.original_client_socket_id,
+ });
+ }
+ }})
+ }catch(error){
+ console.log(error)
+ return res.status(500).send({message: error.message});
+ }
+
+ file.buffer = file.data;
+
+ // get file type
+ await (async () => {
+ const { fileTypeFromBuffer } = await import('file-type');
+ const type = await fileTypeFromBuffer(file.buffer);
+ // set file type
+ if(type)
+ file.type = type.mime;
+ else
+ file.type = 'application/octet-stream';
+ })();
+
+
+ // get file name
+ let filename;
+
+ // extract file name from url
+ if(!req.body.name){
+ filename = req.body.name ?? req.body.url.split('/').pop().split('#')[0].split('?')[0];
+ // extension?
+ if(!path.extname(filename)){
+ // get extension from mime type
+ const ext = mime.extension(file.type);
+ // add extension to filename
+ if(ext)
+ filename += '.' + ext;
+ }
+ }else
+ filename = req.body.name;
+
+ // Setup metadata in context
+ {
+ const x = Context.get();
+ const operationTraceSvc = x.get('services').get('operationTrace');
+ const frame = (await operationTraceSvc.add_frame('api:/download'))
+ .attr('gui_metadata', {
+ original_client_socket_id: req.body.original_client_socket_id,
+ socket_id: req.body.socket_id,
+ operation_id: req.body.operation_id,
+ item_upload_id: req.body.item_upload_id,
+ user_id: req.user.id,
+ })
+ ;
+ x.set(operationTraceSvc.ckey('frame'), frame);
+ }
+
+ // write file
+ try{
+ const dirNode = req.values.fsNode;
+ const hl_write = new HLWrite();
+ const response = await hl_write.run({
+ destination_or_parent: dirNode,
+ specified_name: filename,
+ overwrite: false,
+ dedupe_name: req.body.dedupe_name,
+
+ user: req.user,
+ file: file,
+ });
+ return res.send(response);
+ }catch(error){
+ console.log(error)
+ return res.contentType('application/json').status(500).send(error);
+ }
+});
diff --git a/packages/backend/src/routers/drivers/call.js b/packages/backend/src/routers/drivers/call.js
new file mode 100644
index 00000000..7a83b095
--- /dev/null
+++ b/packages/backend/src/routers/drivers/call.js
@@ -0,0 +1,140 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const eggspress = require("../../api/eggspress");
+const { FileFacade } = require("../../services/drivers/FileFacade");
+const { TypeSpec } = require("../../services/drivers/meta/Construct");
+const { TypedValue } = require("../../services/drivers/meta/Runtime");
+const { Context } = require("../../util/context");
+const { TeePromise } = require("../../util/promise");
+
+let _handle_multipart;
+
+/**
+ * POST /drivers/call
+ *
+ * This endpoint is used to call methods offered by driver interfaces.
+ * The implementation used by each interface depends on the user's
+ * configuration.
+ *
+ * The request body can be a JSON object or multipart/form-data.
+ * For multipart/form-data, the caller must be aware that all fields
+ * are required to be sent before files so that the request handler
+ * and underlying driver implementation can decide what to do with
+ * file streams as they come.
+ *
+ * Example request body:
+ * {
+ * "interface": "puter-ocr",
+ * "method": "recognize",
+ * "args": {
+ * "file": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAB...
+ * }
+ * }
+ */
+module.exports = eggspress('/drivers/call', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const x = Context.get();
+ const svc_driver = x.get('services').get('driver');
+
+ const interface_name = req.body.interface;
+ const test_mode = req.body.test_mode;
+
+ const params = req.headers['content-type'].includes('multipart/form-data')
+ ? await _handle_multipart(req)
+ : req.body.args;
+
+ let context = Context.get();
+ if ( test_mode ) context = context.sub({ test_mode: true });
+
+ const result = await context.arun(async () => {
+ return await svc_driver.call(interface_name, req.body.method, params);
+ });
+
+ _respond(res, result);
+});
+
+const _respond = (res, result) => {
+ if ( result.result instanceof TypedValue ) {
+ const tv = result.result;
+ if ( TypeSpec.adapt({ $: 'stream' }).equals(tv.type) ) {
+ res.set('Content-Type', tv.type.raw.content_type);
+ tv.value.pipe(res);
+ return;
+ }
+
+ // This is the
+ if ( typeof result.value === 'object' ) {
+ result.value.type_fallback = true;
+ }
+ res.json(result.value);
+ return;
+ }
+ res.json(result);
+};
+
+_handle_multipart = async (req) => {
+ const busboy = require('busboy');
+ const { Readable } = require('stream');
+
+ const params = {};
+
+ const bb = new busboy({
+ headers: req.headers,
+ });
+
+ const p_ready = new TeePromise();
+ bb.on('file', (fieldname, stream, details) => {
+ const file_facade = new FileFacade();
+ file_facade.values.set('stream', stream);
+ file_facade.values.set('busboy:details', details);
+ if ( params.hasOwnProperty(fieldname) ) {
+ if ( ! Array.isArray(params[fieldname]) ) {
+ params[fieldname] = [params[fieldname]];
+ }
+ params[fieldname].push(file_facade);
+ } else {
+ params[fieldname] = file_facade;
+ }
+ });
+ bb.on('field', (fieldname, value, details) => {
+ if ( params.hasOwnProperty(fieldname) ) {
+ if ( ! Array.isArray(params[fieldname]) ) {
+ params[fieldname] = [params[fieldname]];
+ }
+ params[fieldname].push(value);
+ } else {
+ params[fieldname] = value;
+ }
+ });
+ bb.on('error', (err) => {
+ p_ready.reject(err);
+ });
+ bb.on('close', () => {
+ p_ready.resolve();
+ });
+
+ req.pipe(bb);
+
+ await p_ready;
+
+ return params;
+}
\ No newline at end of file
diff --git a/packages/backend/src/routers/drivers/list-interfaces.js b/packages/backend/src/routers/drivers/list-interfaces.js
new file mode 100644
index 00000000..be291371
--- /dev/null
+++ b/packages/backend/src/routers/drivers/list-interfaces.js
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const eggspress = require("../../api/eggspress");
+const { Interface } = require("../../services/drivers/meta/Construct");
+const { Context } = require("../../util/context");
+
+module.exports = eggspress('/drivers/list-interfaces', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['GET'],
+}, async (req, res, next) => {
+ const x = Context.get();
+ const svc_driver = x.get('services').get('driver');
+
+ const interfaces_raw = await svc_driver.list_interfaces();
+
+ const interfaces = {};
+ for ( const interface_name in interfaces_raw ) {
+ if ( interfaces_raw[interface_name].no_sdk ) continue;
+ interfaces[interface_name] = (new Interface(
+ interfaces_raw[interface_name],
+ { name: interface_name }
+ )).serialize();
+ }
+
+ res.json(interfaces);
+})
diff --git a/packages/backend/src/routers/drivers/usage.js b/packages/backend/src/routers/drivers/usage.js
new file mode 100644
index 00000000..67fab56b
--- /dev/null
+++ b/packages/backend/src/routers/drivers/usage.js
@@ -0,0 +1,155 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../api/APIError");
+const eggspress = require("../../api/eggspress");
+const { get_app } = require("../../helpers");
+const { UserActorType } = require("../../services/auth/Actor");
+const { DB_READ } = require("../../services/database/consts");
+const { Context } = require("../../util/context");
+const { hash_serializable_object } = require("../../util/datautil");
+
+module.exports = eggspress('/drivers/usage', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['GET'],
+}, async (req, res, next) => {
+ const x = Context.get();
+
+ const actor = x.get('actor');
+
+ // Apps cannot (currently) check usage on behalf of users
+ if ( ! ( actor.type instanceof UserActorType ) ) {
+ throw APIError.create('forbidden');
+ }
+
+ const db = x.get('services').get('database').get(DB_READ, 'drivers');
+
+ const usages = {
+ user: {}, // map[str(iface:method)]{date,count,max}
+ apps: {}, // []{app,map[str(iface:method)]{date,count,max}}
+ app_objects: {},
+ };
+
+ const rows = await db.read(
+ 'SELECT * FROM `service_usage_monthly` WHERE user_id=? ' +
+ 'AND `year` = ? AND `month` = ?',
+ [
+ actor.type.user.id,
+ new Date().getUTCFullYear(),
+ new Date().getUTCMonth() + 1,
+ ]
+ );
+
+ const user_is_verified = actor.type.user.email_confirmed;
+
+ for ( const row of rows ) {
+ const app = await get_app({ id: row.app_id });
+
+ const identifying_fields = {
+ service: row.extra,
+ year: row.year,
+ month: row.month,
+ };
+ const user_usage_key = hash_serializable_object(identifying_fields);
+
+ if ( ! usages.user[user_usage_key] ) {
+ usages.user[user_usage_key] = {
+ ...identifying_fields,
+ };
+
+ const method_key = row.extra['driver.implementation'] +
+ ':' + row.extra['driver.method'];
+ const sla_key = `driver:impl:${method_key}`;
+
+ const svc_sla = x.get('services').get('sla');
+ const sla = await svc_sla.get(
+ user_is_verified ? 'user_verified' : 'user_unverified',
+ sla_key
+ );
+
+ usages.user[user_usage_key].monthly_limit =
+ sla?.monthly_limit || null;
+ }
+
+ usages.user[user_usage_key].monthly_usage =
+ (usages.user[user_usage_key].monthly_usage || 0) + row.count;
+
+ // const user_method_usage = usages.user.find(
+ // u => u.key === row.key
+ // );
+
+ // This will be
+ if ( ! app ) continue;
+
+ const app_usages = usages.apps[app.uid]
+ ?? (usages.apps[app.uid] = {});
+
+ const id_plus_app = {
+ ...identifying_fields,
+ app_uid: app.uid,
+ };
+
+ usages.app_objects[app.uid] = app;
+
+ const app_usage_key = hash_serializable_object(id_plus_app);
+
+ if ( ! app_usages[app_usage_key] ) {
+ app_usages[app_usage_key] = {
+ ...identifying_fields,
+ };
+
+ const method_key = row.extra['driver.implementation'] +
+ ':' + row.extra['driver.method'];
+ const sla_key = `driver:impl:${method_key}`;
+
+ const svc_sla = x.get('services').get('sla');
+ const sla = await svc_sla.get('app_default', sla_key);
+
+ app_usages[app_usage_key].monthly_limit =
+ sla?.monthly_limit || null;
+ }
+
+ // TODO: DRY
+ // remove some privileged information
+ delete app.id;
+ delete app.approved_for_listing;
+ delete app.approved_for_opening_items;
+ delete app.godmode;
+ delete app.owner_user_id;
+
+ if ( ! app_usages[app_usage_key] ) {
+ app_usages[app_usage_key] = {};
+ }
+
+ app_usages[app_usage_key].monthly_usage =
+ (app_usages[app_usage_key].monthly_usage || 0) + row.count;
+
+ // usages.apps.push(usage);
+ }
+
+ for ( k in usages.apps ) {
+ usages.apps[k] = Object.values(usages.apps[k]);
+ }
+
+ res.json({
+ user: Object.values(usages.user),
+ apps: usages.apps,
+ app_objects: usages.app_objects,
+ });
+})
diff --git a/packages/backend/src/routers/drivers/xd.js b/packages/backend/src/routers/drivers/xd.js
new file mode 100644
index 00000000..e413fefb
--- /dev/null
+++ b/packages/backend/src/routers/drivers/xd.js
@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const eggspress = require("../../api/eggspress");
+
+const init_client_js = code => {
+ return `
+ document.addEventListener('DOMContentLoaded', function() {
+ (${code})();
+ });
+ `;
+}
+
+const script = async function script () {
+ const call = async ({
+ interface_name,
+ method_name,
+ params,
+ }) => {
+ const response = await fetch('/drivers/call', {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ interface: interface_name,
+ method: method_name,
+ params,
+ }),
+ });
+ return await response.json();
+ };
+
+ const fcall = async ({
+ interface_name,
+ method_name,
+ params,
+ }) => {
+ // multipart request
+ const form = new FormData();
+ form.append('interface', interface_name);
+ form.append('method', method_name);
+ for ( const k in params ) {
+ form.append(k, params[k]);
+ }
+ const response = await fetch('/drivers/call', {
+ method: 'POST',
+ body: form,
+ });
+ return await response.json();
+ };
+
+ window.addEventListener('message', async event => {
+ const { id, interface, method, params } = event.data;
+ let has_file = false;
+ for ( const k in params ) {
+ if ( params[k] instanceof File ) {
+ has_file = true;
+ break;
+ }
+ }
+ const result = has_file ? await fcall({
+ interface_name: interface,
+ method_name: method,
+ params,
+ }) : await call({
+ interface_name: interface,
+ method_name: method,
+ params,
+ });
+ const response = {
+ id,
+ result,
+ };
+ event.source.postMessage(response, event.origin);
+ });
+};
+
+/**
+ * POST /drivers/xd
+ *
+ * This endpoint services the document which receives
+ * cross-document messages from the SDK and forwards
+ * them to the Puter Driver API.
+ */
+module.exports = eggspress('/drivers/xd', {
+ auth: true,
+ allowedMethods: ['GET'],
+}, async (req, res, next) => {
+ res.type('text/html');
+ res.send(`
+
+
+
+ Puter Driver API
+
+
+
+
+ `);
+});
+
diff --git a/packages/backend/src/routers/file.js b/packages/backend/src/routers/file.js
new file mode 100644
index 00000000..15cb97f1
--- /dev/null
+++ b/packages/backend/src/routers/file.js
@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const config = require('../config');
+const {validate_signature_auth, get_url_from_req, get_descendants, id2path, get_user, sign_file} = require('../helpers');
+const { DB_WRITE } = require('../services/database/consts');
+
+// -----------------------------------------------------------------------//
+// GET /file
+// -----------------------------------------------------------------------//
+router.get('/file', async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // validate URL signature
+ try{
+ validate_signature_auth(get_url_from_req(req), 'read');
+ }
+ catch(e){
+ console.log(e)
+ return res.status(403).send(e);
+ }
+
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'filesystem');
+ const mime = require('mime-types')
+
+ const uid = req.query.uid;
+ let download = req.query.download ?? false;
+ if(download === 'true' || download === '1' || download === true)
+ download = true;
+
+ // retrieve FSEntry from db
+ const fsentry = await db.read(
+ `SELECT * FROM fsentries WHERE uuid = ? LIMIT 1`, [uid]
+ );
+
+ // FSEntry not found
+ if(!fsentry[0])
+ return res.status(400).send({message: 'No entry found with this uid'})
+
+ // check if item owner is suspended
+ const user = await get_user({id: fsentry[0].user_id});
+ if(user.suspended)
+ return res.status(401).send({error: 'Account suspended'});
+
+ // ---------------------------------------------------------------//
+ // FSEntry is dir
+ // ---------------------------------------------------------------//
+ if(fsentry[0].is_dir){
+ // convert to path
+ const dirpath = await id2path(fsentry[0].id);
+ console.log(dirpath, fsentry[0].user_id)
+ // get all children of this dir
+ const children = await get_descendants(dirpath, await get_user({id: fsentry[0].user_id}), 1);
+ const signed_children = [];
+ if(children.length>0){
+ for(const child of children){
+ // sign file
+ const signed_child = await sign_file(child, 'write');
+ signed_children.push(signed_child);
+ }
+ }
+ // send to client
+ return res.send(signed_children);
+ }
+
+ // force download?
+ if(download)
+ res.attachment(fsentry[0].name);
+
+ // record fsentry owner
+ res.resource_owner = fsentry[0].user_id;
+
+ // try to deduce content-type
+ const contentType = mime.contentType(fsentry[0].name)
+
+ // update `accessed`
+ db.write(
+ "UPDATE fsentries SET accessed = ? WHERE `id` = ?",
+ [Date.now()/1000, fsentry[0].id]
+ );
+
+ const range = req.headers.range;
+ //--------------------------------------------------
+ // No range
+ //--------------------------------------------------
+ if (!range) {
+ // set content-type, if available
+ if(contentType !== null)
+ res.setHeader('Content-Type', contentType);
+
+ const storage = req.ctx.get('storage');
+
+ // stream data from S3
+ try{
+ let stream = await storage.create_read_stream({
+ key: fsentry[0].uuid,
+ bucket: fsentry[0].bucket,
+ bucket_region: fsentry[0].bucket_region,
+ });
+ return stream.pipe(res);
+ }catch(e){
+ return res.type('application/json').status(500).send({message: 'There was an internal problem reading the file.'});
+ }
+ }
+ //--------------------------------------------------
+ // Range
+ //--------------------------------------------------
+ else{
+ // get file size
+ const file_size = fsentry[0].size;
+ const total = fsentry[0].size;
+ const user_agent = req.get('User-Agent');
+
+ let start, end, CHUNK_SIZE = 5000000;
+ let is_safari = false;
+
+ // Parse range header
+ var parts = range.replace(/bytes=/, "").split("-");
+ var partialstart = parts[0];
+ var partialend = parts[1];
+
+ start = parseInt(partialstart, 10);
+ end = partialend ? parseInt(partialend, 10) : total-1;
+
+ // Safari
+ if(user_agent && user_agent.toLowerCase().includes('safari') && !user_agent.includes('Chrome')){
+ is_safari = true;
+ CHUNK_SIZE = (end-start)+1;
+ }
+ // All other user agents
+ else{
+ end = Math.min(start + CHUNK_SIZE, file_size - 1);
+ }
+
+ // Create headers
+ const headers = {
+ "Content-Range": `bytes ${start}-${end}/${file_size}`,
+ "Accept-Ranges": "bytes",
+ "Content-Length": is_safari ? CHUNK_SIZE : (end-start+1),
+ };
+
+ // Set Content-Type, if available
+ if(contentType)
+ headers["Content-Type"] = contentType;
+
+ // HTTP Status 206 for Partial Content
+ res.writeHead(206, headers);
+
+ // init S3
+ const s3 = new AWS.S3({
+ accessKeyId: config.s3_access_key,
+ secretAccessKey: config.s3_secret_key,
+ region: fsentry[0].bucket_region,
+ });
+
+ try{
+ let stream = await storage.create_read_stream({
+ key: fsentry[0].uuid,
+ bucket: fsentry[0].bucket,
+ bucket_region: fsentry[0].bucket_region,
+ });
+ return stream.pipe(res);
+ }catch(e){
+ console.log(e)
+ return res.type('application/json').status(500).send({message: 'There was an internal problem reading the file.'});
+ }
+ }
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/filesystem_api/batch/PathResolver.js b/packages/backend/src/routers/filesystem_api/batch/PathResolver.js
new file mode 100644
index 00000000..a403c5d6
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/batch/PathResolver.js
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require('../../../api/APIError.js');
+const { relativeSelector, NodeUIDSelector } = require('../../../filesystem/node/selectors.js');
+const ERR_INVALID_PATHREF = 'Invalid path reference in path: ';
+const ERR_UNKNOWN_PATHREF = 'Unknown path reference in path: ';
+
+/**
+ * Resolves path references in batch requests.
+ *
+ * A path reference is a path that starts with a dollar sign ($).
+ * It will resolve to the path that was returned by the operation
+ * with the same name in its `as` field.
+ *
+ * For example, if the operation `mkdir` has an `as` field with the
+ * value `newdir`, then the path `$newdir` will resolve to the path
+ * that was returned by the `mkdir` operation.
+ */
+module.exports = class PathResolver {
+ constructor ({ user }) {
+ this.references = {};
+ this.selectors = {};
+ this.meta = {};
+ this.user = user;
+
+ this.listeners = {};
+
+ this.log = globalThis.services.get('log-service').create('path-resolver');
+ }
+
+ /**
+ * putPath - Add a path reference.
+ *
+ * The path reference will be resolved to the given path.
+ *
+ * @param {string} refName - The name of the path reference.
+ * @param {string} path - The path to resolve to.
+ */
+ putPath (refName, path) {
+ this.references[refName] = { path };
+ }
+
+ putSelector (refName, selector, meta) {
+ this.log.debug(`putSelector called for: ${refName}`)
+ this.selectors[refName] = selector;
+ this.meta[refName] = meta;
+ if ( ! this.listeners.hasOwnProperty(refName) ) return;
+
+ for ( const lis of this.listeners[refName] ) lis();
+ }
+
+ /**
+ * resolve - Resolve a path reference.
+ *
+ * If the given path does not start with a dollar sign ($),
+ * it will be returned as-is. Otherwise, the path reference
+ * will be resolved to the path that was given to `putPath`.
+ *
+ * @param {string} inputPath
+ * @returns {string} The resolved path.
+ */
+
+ resolve (inputPath) {
+ const refName = this.getReferenceUsed(inputPath);
+ if ( refName === null ) return inputPath;
+ if ( ! this.references.hasOwnProperty(refName) ) {
+ throw APIError.create(400, ERR_UNKNOWN_PATHREF + refName);
+ }
+
+ return this.references[refName].path +
+ inputPath.substring(refName.length + 1);
+ }
+
+ async awaitSelector (inputPath) {
+ if ( inputPath.startsWith('~/') ) {
+ return `/${this.user.username}/${inputPath.substring(2)}`;
+ }
+ if ( inputPath === '~' ) {
+ return `/${this.user.username}`;
+ }
+ const refName = this.getReferenceUsed(inputPath);
+ if ( refName === null ) return inputPath;
+
+ this.log.debug(`-- awaitSelector -- input path is ${inputPath}`);
+ this.log.debug(`-- awaitSelector -- refName is ${refName}`);
+ if ( ! this.selectors.hasOwnProperty(refName) ) {
+ this.log.debug(`-- awaitSelector -- doing the await`);
+ if ( ! this.listeners[refName] ) {
+ this.listeners[refName] = [];
+ }
+ await new Promise (rslv => {
+ this.listeners[refName].push(rslv);
+ });
+ }
+
+ const subpath = inputPath.substring(refName.length + 1);
+ const selector = this.selectors[refName];
+
+ return relativeSelector(selector, subpath);
+ }
+
+ getMeta (inputPath) {
+ const refName = this.getReferenceUsed(inputPath);
+ if ( refName === null ) return null;
+
+ return this.meta[refName];
+ }
+
+ getReferenceUsed (inputPath) {
+ if ( ! inputPath.startsWith('$') ) return null;
+
+ const endOfRefName = inputPath.includes('/')
+ ? inputPath.indexOf('/', 1) : inputPath.length;
+ const refName = inputPath.substring(1, endOfRefName);
+
+ if ( refName === '' ) {
+ throw APIError.create(400, ERR_INVALID_PATHREF + inputPath);
+ }
+
+ return refName;
+ }
+}
diff --git a/packages/backend/src/routers/filesystem_api/batch/all.js b/packages/backend/src/routers/filesystem_api/batch/all.js
new file mode 100644
index 00000000..53b05fb4
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/batch/all.js
@@ -0,0 +1,299 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../../api/APIError");
+const eggspress = require("../../../api/eggspress");
+const config = require("../../../config");
+const PathResolver = require("./PathResolver");
+const { WorkUnit } = require("../../../services/runtime-analysis/ExpectationService");
+const { Context } = require("../../../util/context");
+const Busboy = require('busboy');
+const { BatchExecutor } = require("../../../filesystem/batch/BatchExecutor");
+const { TeePromise } = require("../../../util/promise");
+const { EWMA, MovingMode } = require("../../../util/opmath");
+
+const commands = require('../../../filesystem/batch/commands.js').commands;
+
+module.exports = eggspress('/batch', {
+ subdomain: 'api',
+ verified: true,
+ auth2: true,
+ fs: true,
+ // json: true,
+ // files: ['file'],
+ // multest: true,
+ // multipart_jsons: ['operation'],
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const log = req.services.get('log-service').create('batch');
+ const errors = req.services.get('error-service').create(log);
+
+ const x = Context.get();
+ x.set('dbrr_channel', 'batch');
+
+ let app;
+ if ( req.body.app_uid ) {
+ app = await get_app({uid: req.body.app_uid})
+ }
+
+ const expected_metadata = {
+ original_client_socket_id: undefined,
+ socket_id: undefined,
+ operation_id: undefined,
+ };
+
+ // Errors not within operations that can only be detected
+ // while the request is streaming will be assigned to this
+ // value.
+ let request_errors_ = [];
+
+ let frame;
+ const create_frame = () => {
+ const operationTraceSvc = x.get('services').get('operationTrace');
+ frame = operationTraceSvc.add_frame_sync('api:/batch', x)
+ .attr('gui_metadata', {
+ ...expected_metadata,
+ user_id: req.user.id,
+ })
+ ;
+ x.set(operationTraceSvc.ckey('frame'), frame);
+
+ const svc_clientOperation = x.get('services').get('client-operation');
+ const tracker = svc_clientOperation.add_operation({
+ name: 'batch',
+ tags: ['fs'],
+ frame,
+ metadata: {
+ user_id: req.user.id,
+ }
+ });
+ x.set(svc_clientOperation.ckey('tracker'), tracker);
+ }
+
+ // Make sure usage is cached
+ await req.fs.sizeService.get_usage(req.user.id);
+
+ globalThis.average_chunk_size = new MovingMode({
+ alpha: 0.7,
+ initial: 1,
+ });
+
+ const batch_widget = {
+ ic: 0,
+ ops: 0,
+ sc: 0,
+ ec: 0,
+ wc: 0,
+ output () {
+ let s = `Batch Operation: ${this.ic}`;
+ s += `; oc = ${this.ops}`;
+ s += `; sc = ${this.sc}`;
+ s += `; ec = ${this.ec}`;
+ s += `; wc = ${this.wc}`;
+ s += `; cz = ${globalThis.average_chunk_size.get()}`;
+ return s;
+ }
+ };
+ if ( config.env == 'dev' ) {
+ const svc_devConsole = x.get('services').get('dev-console');
+ svc_devConsole.remove_widget('batch');
+ svc_devConsole.add_widget(batch_widget.output.bind(batch_widget), "batch");
+ x.set('dev_batch-widget', batch_widget);
+ }
+
+ //-------------------------------------------------------------
+ // Variables used by busboy callbacks
+ //-------------------------------------------------------------
+ // --- library
+ const operation_requires_file = op_spec => {
+ if ( op_spec.op === 'write' ) return true;
+ return false;
+ }
+ const batch_exe = new BatchExecutor(x, {
+ log, errors,
+ user: req.user,
+ });
+ // --- state
+ const pending_operations = [];
+ const response_promises = [];
+ const fileinfos = [];
+ let total = 0;
+ let total_tbd = true;
+
+ const on_first_file = () => {
+ // log fileinfos
+ console.log('HERE ARE THE FILEINFOS');
+ console.log(JSON.stringify(fileinfos, null, 2));
+ }
+
+
+ //-------------------------------------------------------------
+ // Multipart processing (using busboy)
+ //-------------------------------------------------------------
+ const busboy = Busboy({
+ headers: req.headers,
+ });
+
+ const still_reading = new TeePromise();
+
+ busboy.on('field', (fieldname, value, details) => {
+ if ( details.fieldnameTruncated ) {
+ throw new Error('fieldnameTruncated');
+ }
+ if ( details.valueTruncated ) {
+ throw new Error('valueTruncated');
+ }
+
+ if ( expected_metadata.hasOwnProperty(fieldname) ) {
+ expected_metadata[fieldname] = value;
+ req.body[fieldname] = value;
+ return;
+ }
+
+ if ( fieldname === 'fileinfo' ) {
+ fileinfos.push(JSON.parse(value));
+ return;
+ }
+
+ if ( ! frame ) {
+ create_frame();
+ }
+
+ if ( fieldname === 'operation' ) {
+ const op_spec = JSON.parse(value);
+ batch_exe.total++;
+ if ( operation_requires_file(op_spec) ) {
+ console.log(`WAITING FOR FILE ${op_spec.op}`)
+ pending_operations.push(op_spec);
+ response_promises.push(null);
+ return;
+ }
+
+ console.log(`EXEUCING OP ${op_spec.op}`)
+ response_promises.push(
+ batch_exe.exec_op(req, op_spec)
+ );
+ return;
+ }
+
+ req.body[fieldname] = value;
+ });
+
+ let i = 0;
+ let ended = [];
+ let ps = [];
+
+ busboy.on('file', async (fieldname, stream, detais) => {
+ if (false) {
+ ended[i] = false;
+ ps[i] = new TeePromise();
+ const this_i = i;
+ stream.on('end', () => {
+ ps[this_i].resolve();
+ ended[this_i] = true;
+ batch_widget.ec++;
+ });
+ if ( i > 0 ) {
+ if ( ! ended[i-1] ) {
+ batch_widget.sc++;
+ // stream.pause();
+ batch_widget.wc++;
+ await Promise.all(Array(i).fill(0).map((_, j) => ps[j]));
+ batch_widget.wc--;
+ // stream.resume();
+ }
+ }
+ i++;
+ }
+
+ if ( batch_exe.total_tbd ) {
+ batch_exe.total_tbd = false;
+ batch_widget.ic = pending_operations.length;
+ on_first_file();
+ }
+ console.log(`GOT A FILE`)
+
+ if ( fileinfos.length == 0 ) {
+ request_errors_.push(
+ new APIError('batch_too_many_files')
+ );
+ stream.on('data', () => {});
+ stream.on('end', () => {
+ stream.destroy();
+ });
+ return;
+ }
+
+ const file = fileinfos.shift();
+ file.stream = stream;
+
+ if ( pending_operations.length == 0 ) {
+ request_errors_.push(
+ new APIError('batch_too_many_files')
+ );
+ // Elimiate the stream
+ stream.on('data', () => {});
+ stream.on('end', () => {
+ stream.destroy();
+ });
+ console.log('DISCARDED A FILE');
+ return;
+ }
+
+ const op_spec = pending_operations.shift();
+
+ // index in response_promises is first null value
+ const index = response_promises.findIndex(p => p === null);
+ response_promises[index] = batch_exe.exec_op(req, op_spec, file);
+ // response_promises[index] = Promise.resolve(out);
+ });
+
+ busboy.on('close', () => {
+ console.log('GOT DONE READING');
+ still_reading.resolve();
+ });
+
+ req.pipe(busboy);
+
+ //-------------------------------------------------------------
+ // Awaiting responses
+ //-------------------------------------------------------------
+ await still_reading;
+ log.noticeme('WAITING ON OPERATIONS')
+ let responsePromises = response_promises;
+ // let responsePromises = batch_exe.responsePromises;
+ const results = await Promise.all(responsePromises);
+ log.noticeme('RESPONSE GETS SENT!');
+
+ frame.done();
+
+ if ( pending_operations.length ) {
+ for ( const op_spec of pending_operations ) {
+ const err = new APIError('batch_missing_file');
+ request_errors_.push(err);
+ }
+ }
+
+ if ( request_errors_ ) {
+ results.push(...request_errors_.map(e => {
+ return e.serialize();
+ }));
+ }
+
+ res.status(batch_exe.hasError ? 218 : 200).send({ results });
+});
\ No newline at end of file
diff --git a/packages/backend/src/routers/filesystem_api/batch/prepare.js b/packages/backend/src/routers/filesystem_api/batch/prepare.js
new file mode 100644
index 00000000..f3cb5950
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/batch/prepare.js
@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
diff --git a/packages/backend/src/routers/filesystem_api/copy.js b/packages/backend/src/routers/filesystem_api/copy.js
new file mode 100644
index 00000000..bc070cc6
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/copy.js
@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const eggspress = require('../../api/eggspress.js');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam.js');
+const _path = require('path');
+const { NodeUIDSelector } = require('../../filesystem/node/selectors.js');
+const { HLCopy } = require('../../filesystem/hl_operations/hl_copy.js');
+const { Context } = require('../../util/context.js');
+const { DatabaseFSEntryService } = require('../../filesystem/storage/DatabaseFSEntryService.js');
+const { ProxyContainer } = require('../../services/Container.js');
+
+// -----------------------------------------------------------------------//
+// POST /copy
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/copy', {
+ subdomain: 'api',
+ auth2: true,
+ verified: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['POST'],
+ parameters: {
+ source: new FSNodeParam('source'),
+ destination: new FSNodeParam('destination'),
+ }
+}, async (req, res, next) => {
+ const user = req.user
+ const dedupe_name =
+ req.body.dedupe_name ??
+ req.body.change_name ?? false;
+
+ // // check if source would be an ancestor of destination
+ // if((abs_dest_path + '/').startsWith(abs_source_path + '/')){
+ // return res.status(400).send('Can not copy a item into itself.')
+ // }
+ let frame;
+ {
+ const x = Context.get();
+ const operationTraceSvc = x.get('services').get('operationTrace');
+ frame = (await operationTraceSvc.add_frame('api:/copy'))
+ .attr('gui_metadata', {
+ original_client_socket_id: req.body.original_client_socket_id,
+ socket_id: req.body.socket_id,
+ operation_id: req.body.operation_id,
+ user_id: req.user.id,
+ item_upload_id: req.body.item_upload_id,
+ })
+ ;
+ x.set(operationTraceSvc.ckey('frame'), frame);
+ }
+
+ // TEMP: Testing copy with its own sql queue
+ if ( false ) {
+ const x = Context.get();
+ const svc = new ProxyContainer(x.get('services'));
+ const s = new DatabaseFSEntryService({
+ services: x.get('services'),
+ label: 'Copy-DatabaseFSEntryService',
+ });
+ svc.set('systemFSEntryService', s);
+ x.set('services', svc);
+ }
+
+ const log = req.services.get('log-service').create('copy');
+ const filesystem = req.services.get('filesystem');
+
+ // copy
+ const {get_app, uuid2fsentry, is_shared_with_anyone, suggest_app_for_fsentry} = require('../../helpers.js')
+ let new_fsentries = [];
+
+ const tracer = req.services.get('traceService').tracer;
+ await tracer.startActiveSpan('filesystem_api.copy', async span => {
+ // const op = await filesystem.cp(req.fs, {
+ // source: req.values.source,
+ // destinationOrParent: req.values.destination,
+ // user: user,
+ // new_name: req.body.new_name,
+ // overwrite: req.body.overwrite ?? false,
+ // dedupe_name,
+ // });
+
+ // === upcoming copy behaviour ===
+ const hl_copy = new HLCopy();
+ const response = await hl_copy.run({
+ destination_or_parent: req.values.destination,
+ source: req.values.source,
+ new_name: req.body.new_name,
+
+ new_name: req.body.new_name,
+ overwrite: req.body.overwrite ?? false,
+ dedupe_name,
+
+ user: user,
+ });
+
+ span.end();
+ frame.done();
+ return res.send([ response ]);
+ });
+
+ // res.send(new_fsentries)
+});
diff --git a/packages/backend/src/routers/filesystem_api/delete.js b/packages/backend/src/routers/filesystem_api/delete.js
new file mode 100644
index 00000000..076ff79a
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/delete.js
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const config = require('../../config.js');
+const eggspress = require('../../api/eggspress.js');
+const { HLRemove } = require('../../filesystem/hl_operations/hl_remove.js');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam.js');
+
+// -----------------------------------------------------------------------//
+// POST /delete
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/delete', {
+ subdomain: 'api',
+ auth2: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ const user = req.user;
+ const paths = req.body.paths;
+ const recursive = req.body.recursive ?? false;
+ const descendants_only = req.body.descendants_only ?? false;
+
+ if(paths === undefined)
+ return res.status(400).send('paths is required')
+ else if(!Array.isArray(paths))
+ return res.status(400).send('paths must be an array')
+ else if(paths.length === 0)
+ return res.status(400).send('paths cannot be empty')
+
+ const socketio = require('../../socketio.js').getio();
+
+ // try to delete each path in the array one by one (if glob, resolve first)
+ // TODO: remove this pseudo-batch
+ for(let j=0; j < paths.length; j++){
+ let item_path = paths[j];
+ const target = await (new FSNodeParam('path')).consolidate({
+ req: { fs: req.fs, user },
+ getParam: () => paths[j],
+ });
+ const hl_remove = new HLRemove();
+ await hl_remove.run({
+ target,
+ user,
+ recursive,
+ descendants_only,
+ });
+
+ // send realtime success msg to client
+ if(socketio){
+ socketio.to(req.user.id).emit('item.removed', {path: item_path, descendants_only: descendants_only})
+ }
+ }
+
+ res.send({});
+});
diff --git a/packages/backend/src/routers/filesystem_api/mkdir.js b/packages/backend/src/routers/filesystem_api/mkdir.js
new file mode 100644
index 00000000..c3e2392c
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/mkdir.js
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const config = require('../../config');
+const eggspress = require('../../api/eggspress');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam');
+const { HLMkdir } = require('../../filesystem/hl_operations/hl_mkdir');
+const { Context } = require('../../util/context');
+const { boolify } = require('../../util/hl_types');
+
+// -----------------------------------------------------------------------//
+// POST /mkdir
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/mkdir', {
+ subdomain: 'api',
+ verified: true,
+ auth2: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['POST'],
+ parameters: {
+ parent: new FSNodeParam('parent', { optional: true }),
+ shortcut_to: new FSNodeParam('shortcut_to', { optional: true }),
+ }
+}, async (req, res, next) => {
+ // validation
+ if(req.body.path === undefined)
+ return res.status(400).send({message: 'path is required'})
+ else if(req.body.path === '')
+ return res.status(400).send({message: 'path cannot be empty'})
+ else if(req.body.path === null)
+ return res.status(400).send({message: 'path cannot be null'})
+ else if(typeof req.body.path !== 'string')
+ return res.status(400).send({message: 'path must be a string'})
+
+ const overwrite = req.body.overwrite ?? false;
+
+ // modules
+ let frame;
+ {
+ const x = Context.get();
+ const operationTraceSvc = x.get('services').get('operationTrace');
+ frame = (await operationTraceSvc.add_frame('api:/mkdir'))
+ .attr('gui_metadata', {
+ original_client_socket_id: req.body.original_client_socket_id,
+ operation_id: req.body.operation_id,
+ user_id: req.user.id,
+ })
+ ;
+ x.set(operationTraceSvc.ckey('frame'), frame);
+ }
+
+ // PEDANTRY: in theory there's no difference between creating an object just to call
+ // a method on it and calling a utility function. HLMkdir is a class because
+ // it uses traits and supports dependency injection, but those features are
+ // not concerns of this endpoint handler.
+ const hl_mkdir = new HLMkdir();
+ const response = await hl_mkdir.run({
+ parent: req.values.parent,
+ path: req.body.path,
+ overwrite: overwrite,
+ dedupe_name: req.body.dedupe_name ?? false,
+ create_missing_parents: boolify(
+ req.body.create_missing_ancestors ??
+ req.body.create_missing_parents
+ ),
+ user: req.user,
+ shortcut_to: req.values.shortcut_to,
+ });
+
+ // TODO: maybe endpoint handlers are operations too. It would be much
+ // nicer to not have to explicitly call frame.done() here.
+ frame.done();
+
+ return res.send(response);
+})
diff --git a/packages/backend/src/routers/filesystem_api/move.js b/packages/backend/src/routers/filesystem_api/move.js
new file mode 100644
index 00000000..d90016e0
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/move.js
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const eggspress = require('../../api/eggspress.js');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam.js');
+const { HLMove } = require('../../filesystem/hl_operations/hl_move.js');
+const { Context } = require('../../util/context.js');
+
+// -----------------------------------------------------------------------//
+// POST /move
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/move', {
+ subdomain: 'api',
+ auth2: true,
+ verified: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['POST'],
+ parameters: {
+ source: new FSNodeParam('source'),
+ destination: new FSNodeParam('destination'),
+ }
+}, async (req, res, next) => {
+ const dedupe_name =
+ req.body.dedupe_name ??
+ req.body.change_name ?? false;
+
+ let frame;
+ {
+ const x = Context.get();
+ const operationTraceSvc = x.get('services').get('operationTrace');
+ frame = (await operationTraceSvc.add_frame('api:/move'))
+ .attr('gui_metadata', {
+ original_client_socket_id: req.body.original_client_socket_id,
+ socket_id: req.body.socket_id,
+ operation_id: req.body.operation_id,
+ user_id: req.user.id,
+ item_upload_id: req.body.item_upload_id,
+ })
+ ;
+ x.set(operationTraceSvc.ckey('frame'), frame);
+ }
+
+ const tracer = req.services.get('traceService').tracer;
+ await tracer.startActiveSpan('filesystem_api.move', async span => {
+ const hl_move = new HLMove();
+ const response = await hl_move.run({
+ destination_or_parent: req.values.destination,
+ source: req.values.source,
+ user: req.user,
+ new_name: req.body.new_name,
+ overwrite: req.body.overwrite ?? false,
+ dedupe_name,
+ new_metadata: req.body.new_metadata,
+ create_missing_parents: req.body.create_missing_parents ?? false,
+ });
+
+ span.end();
+ frame.done();
+ res.send(response);
+ });
+})
diff --git a/packages/backend/src/routers/filesystem_api/read.js b/packages/backend/src/routers/filesystem_api/read.js
new file mode 100644
index 00000000..8536f552
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/read.js
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const APIError = require('../../api/APIError.js');
+const eggspress = require('../../api/eggspress');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam');
+const { HLRead } = require('../../filesystem/hl_operations/hl_read');
+
+module.exports = eggspress('/read', {
+ subdomain: 'api',
+ auth2: true,
+ verified: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['GET'],
+ alias: {
+ path: 'file',
+ uid: 'file',
+ },
+ parameters: {
+ fsNode: new FSNodeParam('file')
+ }
+}, async (req, res, next) => {
+ const line_count = !req.query.line_count ? undefined : parseInt(req.query.line_count);
+ const byte_count = !req.query.byte_count ? undefined : parseInt(req.query.byte_count);
+ const offset = !req.query.offset ? undefined : parseInt(req.query.offset);
+
+ if (line_count && (!Number.isInteger(line_count) || line_count < 1)) {
+ throw new APIError(400, '`line_count` must be a positive integer');
+ }
+ if (byte_count && (!Number.isInteger(byte_count) || byte_count < 1) ){
+ throw new APIError(400, '`byte_count` must be a positive integer');
+ }
+ if (offset && (!Number.isInteger(offset) || offset < 0)) {
+ throw new APIError(400, '`offset` must be a positive integer');
+ }
+ if (byte_count && line_count) {
+ throw new APIError(400, 'cannot use both line_count and byte_count');
+ }
+
+ if (offset && !byte_count) {
+ throw APIError.create('field_only_valid_with_other_field', null, {
+ key: 'offset',
+ other_key: 'byte_count',
+ });
+ }
+
+ const hl_read = new HLRead();
+ const stream = await hl_read.run({
+ fsNode: req.values.fsNode,
+ user: req.user,
+ line_count,
+ byte_count,
+ offset,
+ version_id: req.query.version_id,
+ });
+
+ res.set('Content-Type', 'application/octet-stream');
+
+ stream.pipe(res);
+
+ return;
+
+ const filesystem = req.services.get('filesystem');
+ await filesystem.read(req.fs, res, {
+ ...req.values,
+ user: req.user,
+ version_id: req.query.version_id,
+ line_count,
+ byte_count,
+ });
+});
diff --git a/packages/backend/src/routers/filesystem_api/readdir.js b/packages/backend/src/routers/filesystem_api/readdir.js
new file mode 100644
index 00000000..f8724a53
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/readdir.js
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../../middleware/auth.js');
+const config = require('../../config.js');
+const PerformanceMonitor = require('../../monitor/PerformanceMonitor.js');
+const { Context } = require('../../util/context.js');
+const eggspress = require('../../api/eggspress.js');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam.js');
+const FlagParam = require('../../api/filesystem/FlagParam.js');
+const { LLReadDir } = require('../../filesystem/ll_operations/ll_readdir.js');
+const { HLReadDir } = require('../../filesystem/hl_operations/hl_readdir.js');
+
+// -----------------------------------------------------------------------//
+// POST /readdir
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/readdir', {
+ subdomain: 'api',
+ auth2: true,
+ verified: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['POST'],
+ alias: { uid: 'path' },
+ parameters: {
+ subject: new FSNodeParam('path'),
+ recursive: new FlagParam('recursive', { optional: true }),
+ no_thumbs: new FlagParam('no_thumbs', { optional: true }),
+ no_assocs: new FlagParam('no_assocs', { optional: true }),
+ }
+}, async (req, res, next) => {
+ const monitor = PerformanceMonitor.createContext("router.readdir");
+
+ let log; {
+ const x = Context.get();
+ log = x.get('services').get('log-service').create('readdir');
+ log.info(`readdir: ${req.body.path}`);
+ }
+
+ // // `path` validation
+ // if(req.body.path === undefined)
+ // return res.status(400).send('path is required.')
+ // else if(req.body.path === '')
+ // return res.status(400).send('path cannot be empty.')
+ // else if(req.body.path === null)
+ // return res.status(400).send('path cannot be null.')
+ // else if(typeof req.body.path !== 'string')
+ // return res.status(400).send('path must be a string.')
+
+ // if ( req.body.path.startsWith('~') ) {
+ // const homedir = `/${req.user.username}`;
+ // req.body.path = homedir + req.body.path.slice(1);
+ // }
+
+ // `recursive` validation
+ // if(req.body.recursive !== undefined && typeof req.body.recursive !== 'boolean')
+ // return res.status(400).send('recursive must be a boolean.')
+ // else if(req.body.recursive === undefined)
+ // req.body.recursive = false; // default value
+
+ const subject = req.values.subject;
+ const recursive = req.values.recursive;
+ const no_thumbs = req.values.no_thumbs;
+ const no_assocs = req.values.no_assocs;
+
+ {
+ const fs = require('fs');
+ fs.appendFileSync('/tmp/readdir.log',
+ JSON.stringify({
+ recursive,
+ no_thumbs,
+ no_assocs,
+ }, null, 2) + '\n');
+ }
+
+ const hl_readdir = new HLReadDir();
+ const result = await hl_readdir.run({
+ subject,
+ recursive,
+ no_thumbs,
+ no_assocs,
+ user: req.user,
+ });
+
+ // check for duplicate names
+ if ( ! recursive ) {
+ const names = new Set();
+ for ( const entry of result ) {
+ if ( names.has(entry.name) ) {
+ log.error(`Duplicate name: ${entry.name}`);
+ // throw new Error(`Duplicate name: ${entry.name}`);
+ }
+ names.add(entry.name);
+ }
+ }
+
+ res.send(result);
+ return;
+});
diff --git a/packages/backend/src/routers/filesystem_api/rename.js b/packages/backend/src/routers/filesystem_api/rename.js
new file mode 100644
index 00000000..b2632178
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/rename.js
@@ -0,0 +1,183 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const eggspress = require('../../api/eggspress.js');
+const APIError = require('../../api/APIError.js');
+const { Context } = require('../../util/context.js');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam.js');
+const { DB_WRITE } = require('../../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /rename
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/rename', {
+ subdomain: 'api',
+ auth2: true,
+ verified: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['POST'],
+ alias: { uid: 'path' },
+ parameters: {
+ subject: new FSNodeParam('path'),
+ },
+}, async (req, res, next) => {
+ console.log('ACTIVATED THIS ROUTE');
+
+ if(!req.body.new_name) {
+ throw APIError.create('field_missing', null, {
+ key: 'new_name',
+ });
+ }
+ if (typeof req.body.new_name !== 'string') {
+ throw APIError.create('field_invalid', null, {
+ key: 'new_name',
+ expected: 'string',
+ got: typeof req.body.new_name,
+ });
+ }
+
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'filesystem');
+ const mime = require('mime-types');
+ const {get_app, validate_fsentry_name, uuid2fsentry, chkperm, id2path} = require('../../helpers.js');
+ const _path = require('path');
+
+ // new_name validation
+ try{
+ validate_fsentry_name(req.body.new_name)
+ }catch(e){
+ return res.status(400).send({
+ error:{
+ message: e.message
+ }
+ });
+ }
+
+ const { subject } = req.values;
+
+ //get fsentry
+ if ( ! await subject.exists() ) {
+ throw APIError.create('subject_does_not_exist');
+ }
+
+ // Access control
+ {
+ const actor = Context.get('actor');
+ const svc_acl = Context.get('services').get('acl');
+ if ( ! await svc_acl.check(actor, subject, 'write') ) {
+ throw await svc_acl.get_safe_acl_error(actor, subject, 'write');
+ }
+ }
+
+ await subject.fetchEntry();
+ let fsentry = subject.entry;
+
+ // immutable
+ if(fsentry.immutable){
+ return res.status(400).send({
+ error:{
+ message: 'Immutable: cannot rename.'
+ }
+ })
+ }
+
+ let res1;
+
+ // parent is root
+ if(fsentry.parent_uid === null){
+ try{
+ res1 = await db.read(
+ `SELECT uuid FROM fsentries WHERE parent_uid IS NULL AND name = ? AND id != ? LIMIT 1`,
+ [
+ //name
+ req.body.new_name,
+ await subject.get('mysql-id'),
+ ]);
+ }catch(e){
+ console.log(e)
+ }
+ }
+ // parent is regular dir
+ else{
+ res1 = await db.read(
+ `SELECT uuid FROM fsentries WHERE parent_uid = ? AND name = ? AND id != ? LIMIT 1`,
+ [
+ //parent_uid
+ fsentry.parent_uid,
+ //name
+ req.body.new_name,
+ await subject.get('mysql-id'),
+ ]);
+ }
+ if(res1[0]){
+ throw APIError.create('item_with_same_name_exists', null, {
+ entry_name: req.body.new_name,
+ });
+ }
+
+ const old_path = await id2path(await subject.get('mysql-id'));
+ const new_path = _path.join(_path.dirname(old_path), req.body.new_name);
+
+ // update `name`
+ await db.write(
+ `UPDATE fsentries SET name = ?, path = ? WHERE id = ?`,
+ [req.body.new_name, new_path, await subject.get('mysql-id')]
+ )
+
+ const filesystem = req.services.get('filesystem');
+ await filesystem.update_child_paths(old_path, new_path, req.user.id);
+
+ // associated_app
+ let associated_app;
+ if(fsentry.associated_app_id){
+ const app = await get_app({id: fsentry.associated_app_id})
+ // remove some privileged information
+ delete app.id;
+ delete app.approved_for_listing;
+ delete app.approved_for_opening_items;
+ delete app.godmode;
+ delete app.owner_user_id;
+ // add to array
+ associated_app = app;
+ }else{
+ associated_app = {};
+ }
+
+ // send the fsentry of the new object created
+ const contentType = mime.contentType(req.body.new_name)
+ const return_obj = {
+ uid: req.body.uid,
+ name: req.body.new_name,
+ is_dir: fsentry.is_dir,
+ path: new_path,
+ old_path: old_path,
+ type: contentType ? contentType : null,
+ associated_app: associated_app,
+ original_client_socket_id: req.body.original_client_socket_id,
+ };
+
+ // send realtime success msg to client
+ let socketio = require('../../socketio.js').getio();
+ if(socketio){
+ socketio.to(req.user.id).emit('item.renamed', return_obj)
+ }
+
+ return res.send(return_obj);
+});
diff --git a/packages/backend/src/routers/filesystem_api/stat.js b/packages/backend/src/routers/filesystem_api/stat.js
new file mode 100644
index 00000000..3196371d
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/stat.js
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const eggspress = require('../../api/eggspress.js');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam');
+const { HLStat } = require('../../filesystem/hl_operations/hl_stat.js');
+
+module.exports = eggspress('/stat', {
+ subdomain: 'api',
+ auth2: true,
+ verified: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['GET', 'POST'],
+ alias: {
+ path: 'subject',
+ uid: 'subject',
+ },
+ parameters: {
+ subject: new FSNodeParam('subject'),
+ }
+}, async (req, res, next) => {
+ // modules
+ const hl_stat = new HLStat();
+ const result = await hl_stat.run({
+ subject: req.values.subject,
+ user: req.user,
+ return_subdomains: req.body.return_subdomains,
+ return_permissions: req.body.return_permissions,
+ return_versions: req.body.return_versions,
+ return_size: req.body.return_size,
+ });
+ res.send(result);
+});
diff --git a/packages/backend/src/routers/filesystem_api/token-read.js b/packages/backend/src/routers/filesystem_api/token-read.js
new file mode 100644
index 00000000..2da57f73
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/token-read.js
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const APIError = require('../../api/APIError.js');
+const eggspress = require('../../api/eggspress');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam');
+const { HLRead } = require('../../filesystem/hl_operations/hl_read');
+const { Context } = require('../../util/context');
+const { AccessTokenActorType } = require('../../services/auth/Actor');
+const mime = require('mime-types');
+
+module.exports = eggspress('/token-read', {
+ subdomain: 'api',
+ verified: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['GET'],
+ alias: {
+ path: 'file',
+ uid: 'file',
+ },
+ parameters: {
+ fsNode: new FSNodeParam('file')
+ }
+}, async (req, res, next) => {
+ const line_count = !req.query.line_count ? undefined : parseInt(req.query.line_count);
+ const byte_count = !req.query.byte_count ? undefined : parseInt(req.query.byte_count);
+ const offset = !req.query.offset ? undefined : parseInt(req.query.offset);
+
+ const access_jwt = req.query.token;
+
+ const svc_auth = Context.get('services').get('auth');
+ const actor = await svc_auth.authenticate_from_token(access_jwt);
+
+ if ( ! actor ) {
+ throw new Error('A');
+ throw APIError.create('token_auth_failed');
+ }
+
+ if ( ! (actor.type instanceof AccessTokenActorType) ) {
+ throw new Error('B');
+ throw APIError.create('token_auth_failed');
+ }
+
+ const context = Context.get();
+ context.set('actor', actor);
+ console.log('actor', actor);
+
+ if (line_count && (!Number.isInteger(line_count) || line_count < 1)) {
+ throw new APIError(400, '`line_count` must be a positive integer');
+ }
+ if (byte_count && (!Number.isInteger(byte_count) || byte_count < 1) ){
+ throw new APIError(400, '`byte_count` must be a positive integer');
+ }
+ if (offset && (!Number.isInteger(offset) || offset < 0)) {
+ throw new APIError(400, '`offset` must be a positive integer');
+ }
+ if (byte_count && line_count) {
+ throw new APIError(400, 'cannot use both line_count and byte_count');
+ }
+
+ if (offset && !byte_count) {
+ throw APIError.create('field_only_valid_with_other_field', null, {
+ key: 'offset',
+ other_key: 'byte_count',
+ });
+ }
+
+ const hl_read = new HLRead();
+ const stream = await context.arun(async () => await hl_read.run({
+ fsNode: req.values.fsNode,
+ user: req.user,
+ actor,
+ line_count,
+ byte_count,
+ offset,
+ version_id: req.query.version_id,
+ }));
+
+ const name = await req.values.fsNode.get('name');
+ const mime_type = mime.contentType(name);
+ res.setHeader('Content-Type', mime_type);
+
+ stream.pipe(res);
+
+ return;
+
+ const filesystem = req.services.get('filesystem');
+ await filesystem.read(req.fs, res, {
+ ...req.values,
+ user: req.user,
+ version_id: req.query.version_id,
+ line_count,
+ byte_count,
+ });
+});
+
diff --git a/packages/backend/src/routers/filesystem_api/touch.js b/packages/backend/src/routers/filesystem_api/touch.js
new file mode 100644
index 00000000..f13d1e54
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/touch.js
@@ -0,0 +1,165 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../../middleware/auth.js');
+const config = require('../../config.js');
+const { DB_WRITE } = require('../../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /touch
+// -----------------------------------------------------------------------//
+router.post('/touch', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../../helpers.js').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ const db = req.services.get('database').get(DB_WRITE, 'filesystem');
+ const { v4: uuidv4 } = require('uuid');
+ const _path = require('path');
+ const {convert_path_to_fsentry, validate_fsentry_name, chkperm} = require('../../helpers.js');
+
+ // validation
+ if(req.body.path === undefined)
+ return res.status(400).send('path is required');
+ // path must be a string
+ else if (typeof req.body.path !== 'string')
+ return res.status(400).send('path must be a string.');
+ else if(req.body.path.trim() === '')
+ return res.status(400).send('path cannot be empty');
+
+ const dirpath = _path.dirname(_path.resolve('/', req.body.path))
+ const target_name = _path.basename(_path.resolve('/', req.body.path))
+ const set_accessed_to_now = req.body.set_accessed_to_now
+ const set_modified_to_now = req.body.set_modified_to_now
+
+ // cannot touch in root
+ if(dirpath === '/')
+ return res.status(400).send('Can not touch in root.')
+
+ // name validation
+ try{
+ validate_fsentry_name(target_name)
+ }catch(e){
+ return res.status(400).send(e);
+ }
+
+ // convert dirpath to its fsentry
+ const parent = await convert_path_to_fsentry(dirpath);
+
+ // dirpath not found
+ if(parent === false)
+ return res.status(400).send("Target path not found");
+
+ // check permission
+ if(!await chkperm(parent, req.user.id, 'write'))
+ return res.status(403).send({ code:`forbidden`, message: `permission denied.`})
+
+ // check if a FSEntry with the same name exists under this path
+ const existing_fsentry = await convert_path_to_fsentry(_path.resolve('/', dirpath + '/' + target_name))
+
+ // current epoch
+ const ts = Date.now() / 1000;
+
+ // set_accessed_to_now
+ if(set_accessed_to_now){
+ await db.write(
+ `INSERT INTO fsentries
+ (uuid, parent_uid, user_id, name, is_dir, created, modified, size) VALUES
+ ( ?, ?, ?, ?, false, ?, ?, 0)
+ ON DUPLICATE KEY UPDATE accessed=?`,
+ [
+ //uuid
+ (existing_fsentry !== false) ? existing_fsentry.uuid : uuidv4(),
+ //parent_uid
+ (parent === null) ? null : parent.uuid,
+ //user_id
+ parent === null ? req.user.id : parent.user_id,
+ //name
+ target_name,
+ //created
+ ts,
+ //modified
+ ts,
+ //accessed
+ ts
+ ]
+ );
+ }
+ // set_modified_to_now
+ else if(set_modified_to_now){
+ await db.write(
+ `INSERT INTO fsentries
+ (uuid, parent_uid, user_id, name, is_dir, created, modified, size) VALUES
+ ( ?, ?, ?, ?, false, ?, ?, 0)
+ ON DUPLICATE KEY UPDATE modified=?`,
+ [
+ //uuid
+ (existing_fsentry !== false) ? existing_fsentry.uuid : uuidv4(),
+ //parent_uid
+ (parent === null) ? null : parent.uuid,
+ //user_id
+ parent === null ? req.user.id : parent.user_id,
+ //name
+ target_name,
+ //created
+ ts,
+ //modified
+ ts,
+ //modified
+ ts
+ ]
+ );
+ }else{
+ await db.write(
+ `INSERT INTO fsentries
+ (uuid, parent_uid, user_id, name, is_dir, created, modified, size) VALUES
+ ( ?, ?, ?, ?, false, ?, ?, 0)
+ ON DUPLICATE KEY UPDATE accessed=?, modified=?, created=?`,
+ [
+ //uuid
+ (existing_fsentry !== false) ? existing_fsentry.uuid : uuidv4(),
+ //parent_uid
+ (parent === null) ? null : parent.uuid,
+ //user_id
+ parent === null ? req.user.id : parent.user_id,
+ //name
+ target_name,
+ //created
+ ts,
+ //modified
+ ts,
+ //accessed
+ ts,
+ //modified
+ ts,
+ //created
+ ts,
+ ]
+ );
+ }
+ return res.send('')
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/filesystem_api/write.js b/packages/backend/src/routers/filesystem_api/write.js
new file mode 100644
index 00000000..6c18ae9d
--- /dev/null
+++ b/packages/backend/src/routers/filesystem_api/write.js
@@ -0,0 +1,236 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const eggspress = require('../../api/eggspress.js');
+const FSNodeParam = require('../../api/filesystem/FSNodeParam.js');
+const { HLWrite } = require('../../filesystem/hl_operations/hl_write.js');
+const { boolify } = require('../../util/hl_types.js');
+const { Context } = require('../../util/context.js');
+const Busboy = require('busboy');
+const { TeePromise } = require('../../util/promise.js');
+const APIError = require('../../api/APIError.js');
+const api_error_handler = require('../../api/api_error_handler.js');
+
+// -----------------------------------------------------------------------//
+// POST /up | /write
+// -----------------------------------------------------------------------//
+module.exports = eggspress(['/up', '/write'], {
+ subdomain: 'api',
+ verified: true,
+ auth2: true,
+ fs: true,
+ json: true,
+ allowedMethods: ['POST'],
+ // files: ['file'],
+ // multest: true,
+ alias: { uid: 'path' },
+ // parameters: {
+ // fsNode: new FSNodeParam('path'),
+ // target: new FSNodeParam('shortcut_to', { optional: true }),
+ // }
+}, async (req, res, next) => {
+ // Note: parameters moved here because the parameter
+ // middleware won't work while using busboy
+ const parameters = {
+ fsNode: new FSNodeParam('path'),
+ target: new FSNodeParam('shortcut_to', { optional: true }),
+ };
+
+ // modules
+ const {get_app, mkdir} = require('../../helpers.js')
+
+ // if(!req.files)
+ // return res.status(400).send('No files uploaded');
+
+ // Is this an entry for an app?
+ let app;
+ if ( req.body.app_uid ) {
+ app = await get_app({uid: req.body.app_uid})
+ }
+
+ const x = Context.get();
+ let frame;
+ const frame_meta_ready = async () => {
+ const operationTraceSvc = x.get('services').get('operationTrace');
+ frame = (await operationTraceSvc.add_frame('api:/write'))
+ .attr('gui_metadata', {
+ original_client_socket_id: req.body.original_client_socket_id,
+ socket_id: req.body.socket_id,
+ operation_id: req.body.operation_id,
+ user_id: req.user.id,
+ item_upload_id: req.body.item_upload_id,
+ })
+ ;
+ x.set(operationTraceSvc.ckey('frame'), frame);
+
+ const svc_clientOperation = x.get('services').get('client-operation');
+ const tracker = svc_clientOperation.add_operation({
+ frame,
+ metadata: {
+ user_id: req.user.id,
+ }
+ });
+ x.set(svc_clientOperation.ckey('tracker'), tracker);
+ }
+
+ //-------------------------------------------------------------
+ // Variables used by busboy callbacks
+ //-------------------------------------------------------------
+ const on_first_file = () => {
+ frame_meta_ready();
+ };
+
+ //-------------------------------------------------------------
+ // Multipart processing (using busboy)
+ //-------------------------------------------------------------
+ const busboy = Busboy({ headers: req.headers });
+
+ let uploaded_file = null;
+ const p_ready = new TeePromise();
+
+ busboy.on('field', (fieldname, value, details) => {
+ if ( details.fieldnameTruncated ) {
+ throw new Error('fieldnameTruncated');
+ }
+ if ( details.valueTruncated ) {
+ throw new Error('valueTruncated');
+ }
+
+ req.body[fieldname] = value;
+ });
+
+ busboy.on('file', (fieldname, stream, details) => {
+ const {
+ filename, mimetype,
+ } = details;
+
+ uploaded_file = {
+ size: req.body.size,
+ name: filename,
+ mimetype,
+ stream,
+
+ // TODO: Standardize the fileinfo object
+
+ // thumbnailer expects `mimetype` to be `type`
+ type: mimetype,
+
+ // alias for name, used only in here it seems
+ originalname: filename,
+ };
+
+ p_ready.resolve();
+ });
+
+ busboy.on('error', err => {
+ console.log('GOT ERROR READING', err )
+ p_ready.reject(err);
+ });
+
+ busboy.on('close', () => {
+ console.log('GOT DNE RADINGR')
+ p_ready.resolve();
+ });
+
+ req.pipe(busboy);
+
+ console.log('Awaiting ready');
+ await p_ready;
+ console.log('Done awaiting ready');
+
+ // Copied from eggspress; needed here because we're using busboy
+ for ( const key in parameters ) {
+ const param = parameters[key];
+ if ( ! req.values ) req.values = {};
+
+ const values = req.method === 'GET' ? req.query : req.body;
+ const getParam = (key) => values[key];
+ try {
+ const result = await param.consolidate({ req, getParam });
+ req.values[key] = result;
+ } catch (e) {
+ api_error_handler(e, req, res, next);
+ return;
+ }
+ }
+
+ if ( req.body.size === undefined ) {
+ throw APIError.create('missing_expected_metadata', null, {
+ keys: ['size'],
+ })
+ }
+
+ console.log('TRGET', req.values.target);
+
+ const hl_write = new HLWrite();
+ const response = await hl_write.run({
+ destination_or_parent: req.values.fsNode,
+ specified_name: req.body.name,
+ fallback_name: uploaded_file.originalname,
+ overwrite: await boolify(req.body.overwrite),
+ dedupe_name: await boolify(req.body.dedupe_name),
+ shortcut_to: req.values.target,
+
+ create_missing_parents: boolify(
+ req.body.create_missing_ancestors ??
+ req.body.create_missing_parents
+ ),
+
+ user: req.user,
+ file: uploaded_file,
+
+ app_id: app ? app.id : null,
+ });
+
+ if ( frame ) frame.done();
+ return res.send(response);
+
+ // upload files one by one
+ // for (let index = 0; index < req.files.length; index++) {
+ // let uploaded_file = req.files[index];
+
+ // // TEMP: create stream from buffer
+ // if ( uploaded_file.buffer ) {
+ // uploaded_file = { ...uploaded_file };
+ // const buffer = uploaded_file.buffer;
+ // uploaded_file.stream = (() => {
+ // const { Readable } = require('stream');
+ // return Readable.from(buffer);
+ // })();
+ // delete uploaded_file.buffer;
+ // }
+
+ // const hl_write = new HLWrite();
+ // const response = await hl_write.run({
+ // destination_or_parent: req.values.fsNode,
+ // specified_name: req.body.name,
+ // fallback_name: uploaded_file.originalname,
+ // overwrite: await boolify(req.body.overwrite),
+ // dedupe_name: await boolify(req.body.dedupe_name),
+ // shortcut_to: req.values.target,
+
+ // create_missing_parents: boolify(req.body.create_missing_ancestors),
+
+ // user: req.user,
+ // file: uploaded_file,
+ // });
+
+ // return res.send(response);
+ // }
+});
diff --git a/packages/backend/src/routers/get-dev-profile.js b/packages/backend/src/routers/get-dev-profile.js
new file mode 100644
index 00000000..3b297d0d
--- /dev/null
+++ b/packages/backend/src/routers/get-dev-profile.js
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const config = require('../config.js');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+
+// -----------------------------------------------------------------------//
+// GET /get-dev-profile
+// -----------------------------------------------------------------------//
+router.get('/get-dev-profile', auth, express.json(), async (req, response, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return response.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ try{
+ // auth
+ response.send({
+ first_name: req.user.dev_first_name,
+ last_name: req.user.dev_last_name,
+ approved_for_incentive_program: req.user.dev_approved_for_incentive_program,
+ joined_incentive_program: req.user.dev_joined_incentive_program,
+ paypal: req.user.dev_paypal,
+ });
+ }catch(e){
+ console.log(e)
+ response.status(400).send()
+ }
+})
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/get-launch-apps.js b/packages/backend/src/routers/get-launch-apps.js
new file mode 100644
index 00000000..2664c0dd
--- /dev/null
+++ b/packages/backend/src/routers/get-launch-apps.js
@@ -0,0 +1,149 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+const { get_app } = require('../helpers.js');
+const { DB_READ } = require('../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// GET /get-launch-apps
+// -----------------------------------------------------------------------//
+router.get('/get-launch-apps', auth, express.json(), async (req, res, next)=>{
+ let final_returned_obj = {};
+ let retobj = [];
+ // -----------------------------------------------------------------------//
+ // Recent apps
+ // -----------------------------------------------------------------------//
+ let apps = [];
+
+ const db = req.services.get('database').get(DB_READ, 'apps');
+
+ // First try the cache to see if we have recent apps
+ apps = kv.get('app_opens:user:' + req.user.id);
+
+ // If cache is empty, query the db and update the cache
+ if(!apps || !Array.isArray(apps) || apps.length === 0){
+ apps = await db.read(
+ 'SELECT DISTINCT app_uid FROM app_opens WHERE user_id = ? GROUP BY app_uid ORDER BY MAX(_id) DESC LIMIT 10',
+ [req.user.id]);
+ // Update cache with the results from the db (if any results were returned)
+ if(apps && Array.isArray(apps) && apps.length > 0)
+ kv.set('app_opens:user:' + req.user.id, apps);
+ }
+
+ for (let index = 0; index < apps.length; index++) {
+ const app = await get_app({uid: apps[index].app_uid});
+ let final_obj = {};
+
+ // prepare each app for returning to user by only returning the necessary fields
+ // and adding them to the retobj array
+ if(app){
+ final_obj = {
+ uuid: app.uid,
+ name: app.name,
+ title: app.title,
+ icon: app.icon,
+ godmode: app.godmode,
+ maximize_on_start: app.maximize_on_start,
+ index_url: app.index_url,
+ };
+ }
+ // add to object to be returned
+ retobj.push(final_obj)
+ }
+ final_returned_obj.recent = retobj;
+ // -----------------------------------------------------------------------//
+ // Recommended apps
+ // -----------------------------------------------------------------------//
+ // reset retobj
+ retobj = [];
+ let app_names = [
+ 'app-center',
+ 'dev-center',
+ 'code',
+ 'editor',
+ 'draw',
+ 'camera',
+ 'recorder',
+ 'terminal',
+ 'shell-shockers-outpan',
+ 'krunker',
+ 'rows',
+ 'slash-frvr',
+ 'viewer',
+ 'solitaire-frvr',
+ 'markus',
+ 'player',
+ 'pdf',
+ 'about',
+ 'polotno',
+ 'basketball-frvr',
+ 'gold-digger-frvr',
+ 'plushie-connect',
+ 'hex-frvr',
+ 'spider-solitaire',
+ ]
+
+ // Prepare each app for returning to user by only returning the necessary fields
+ // and adding them to the retobj array
+ if(app_names.length > 0){
+ for (let index = 0; index < app_names.length; index++) {
+ const app = await get_app({name: app_names[index]});
+
+ let final_obj = {};
+ if(app){
+ final_obj = {
+ uuid: app.uid,
+ name: app.name,
+ title: app.title,
+ icon: app.icon,
+ godmode: app.godmode,
+ maximize_on_start: app.maximize_on_start,
+ index_url: app.index_url,
+ };
+ }
+ // add to object to be returned
+ retobj.push(final_obj)
+ }
+
+ // remove duplicates from retobj
+ if(retobj.length > 0)
+ retobj = retobj.filter((obj, pos, arr) => {
+ return arr.map(mapObj => mapObj['name']).indexOf(obj['name']) === pos;
+ })
+ }
+
+ // Order output based on input!
+ let final_obj = [];
+ for (let index = 0; index < app_names.length; index++) {
+ const app_name = app_names[index];
+ for (let index = 0; index < retobj.length; index++) {
+ if(retobj[index].name === app_name)
+ final_obj.push(retobj[index]);
+ }
+ }
+
+ final_returned_obj.recommended = final_obj;
+
+ return res.send(final_returned_obj);
+})
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/hosting/puter-site.js b/packages/backend/src/routers/hosting/puter-site.js
new file mode 100644
index 00000000..40ffb0eb
--- /dev/null
+++ b/packages/backend/src/routers/hosting/puter-site.js
@@ -0,0 +1,197 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const api_error_handler = require("../../api/api_error_handler");
+const config = require("../../config");
+const { get_user, get_app, id2path } = require("../../helpers");
+const { Context } = require("../../util/context");
+const { NodeInternalIDSelector, NodePathSelector } = require("../../filesystem/node/selectors");
+const { TYPE_DIRECTORY } = require("../../filesystem/FSNodeContext");
+const { LLRead } = require("../../filesystem/ll_operations/ll_read");
+const { Actor, UserActorType } = require("../../services/auth/Actor");
+const APIError = require("../../api/APIError");
+
+class PuterSiteMiddleware extends AdvancedBase {
+ static MODULES = {
+ path: require('path'),
+ mime: require('mime-types'),
+ }
+ install (app) {
+ app.use(this.run.bind(this));
+ }
+ async run (req, res, next) {
+ if (
+ ! req.hostname.endsWith(config.static_hosting_domain)
+ && ( req.subdomains[0] !== 'devtest' )
+ ) return next();
+
+ res.setHeader('Access-Control-Allow-Origin', '*');
+
+ try {
+ const expected_ctx = req.ctx;
+ const received_ctx = Context.get();
+
+ if ( expected_ctx && ! received_ctx ) {
+ await expected_ctx.arun(async () => {
+ await this.run_(req, res, next);
+ });
+ } else await this.run_(req, res, next);
+ } catch ( e ) {
+ // TODO: html_error_handler
+ api_error_handler(e, req, res, next);
+ }
+ }
+ async run_ (req, res, next) {
+ const subdomain =
+ req.subdomains[0] === 'devtest' ? 'devtest' :
+ req.hostname.slice(0, -1 * (config.static_hosting_domain.length + 1));
+
+ let path = (req.baseUrl + req.path) ?? 'index.html';
+
+ const context = Context.get();
+ const services = context.get('services');
+
+ const svc_puterSite = services.get('puter-site');
+ const site = await svc_puterSite.get_subdomain(subdomain);
+ if ( site === null ) {
+ return res.status(404).send('Subdomain not found');
+ }
+
+ const subdomain_owner = await get_user({ id: site.user_id });
+ if ( subdomain_owner?.suspended ) {
+ // This used to be "401 Account suspended", but this implies
+ // the client user is suspended, which is not the case.
+ // Instead we simply return 404, indicating that this page
+ // doesn't exist without further specifying that the owner's
+ // account is suspended. (the client user doesn't need to know)
+ return res.status(404).send('Subdomain not found');
+ }
+
+ if (
+ site.associated_app_id &&
+ ! req.query['puter.app_instance_id'] &&
+ ( path === '' || path.endsWith('/') )
+ ) {
+ console.log('ASSOC APP ID', site.associated_app_id);
+ const app = await get_app({ id: site.associated_app_id });
+ return res.redirect(`${config.origin}/app/${app.name}/`);
+ }
+
+ if ( path === '' ) path += '/index.html';
+ else if ( path.endsWith('/') ) path += 'index.html';
+
+ const resolved_url_path =
+ this.modules.path.resolve('/', path);
+
+ const svc_fs = services.get('filesystem');
+
+ let subdomain_root_path = '';
+ if ( site.root_dir_id !== null && site.root_dir_id !== undefined ) {
+ const node = await svc_fs.node(
+ new NodeInternalIDSelector('mysql', site.root_dir_id)
+ );
+ if ( ! await node.exists() ) {
+ res.status(502).send('subdomain is pointing to deleted directory');
+ }
+ if ( await node.get('type') !== TYPE_DIRECTORY ) {
+ res.status(502).send('subdomain is pointing to non-directory');
+ }
+
+ subdomain_root_path = await node.get('path');
+ }
+
+ if ( ! subdomain_root_path || subdomain_root_path === '/' ) {
+ throw new APIError.create('forbidden');
+ }
+
+ const filepath = subdomain_root_path + decodeURIComponent(
+ resolved_url_path
+ );
+
+ const target_node = await svc_fs.node(new NodePathSelector(filepath));
+ await target_node.fetchEntry();
+
+ if ( ! await target_node.exists() ) {
+ return this.respond_index_not_found_(path, req, res, next);
+ }
+
+ const target_is_dir = await target_node.get('type') === TYPE_DIRECTORY;
+
+ if ( target_is_dir && ! resolved_url_path.endsWith('/') ) {
+ return res.redirect(resolved_url_path + '/');
+ }
+
+ if ( target_is_dir ) {
+ return this.respond_index_not_found_(path, req, res, next);
+ }
+
+ const contentType = this.modules.mime.contentType(
+ await target_node.get('name')
+ );
+ res.set('Content-Type', contentType);
+
+ const ll_read = new LLRead();
+ const stream = await ll_read.run({
+ no_acl: true,
+ actor: new Actor({
+ user_uid: req.user ? req.user.uuid : null,
+ type: new UserActorType({ user: req.user }),
+ }),
+ fsNode: target_node,
+ });
+
+ // Destroy the stream if the client disconnects
+ req.on('close', () => {
+ stream.destroy();
+ });
+
+ try {
+ return stream.pipe(res);
+ } catch (e) {
+ return res.status(500).send('Error reading file: ' + e.message);
+ }
+ }
+
+ respond_index_not_found_ (path, req, res, next) {
+ res.status(404);
+ res.set('Content-Type', 'text/html; charset=UTF-8');
+ res.write(``);
+ res.write('
404 ');
+ res.write(`
`)
+ if(path === '/index.html')
+ res.write('index.html
Not Found');
+ else
+ res.write('Not Found');
+ res.write(`
`)
+
+ res.write('
');
+
+ return res.end();
+ }
+}
+
+module.exports = app => {
+ const mw = new PuterSiteMiddleware();
+ mw.install(app);
+};
diff --git a/packages/backend/src/routers/itemMetadata.js b/packages/backend/src/routers/itemMetadata.js
new file mode 100644
index 00000000..1b60fa30
--- /dev/null
+++ b/packages/backend/src/routers/itemMetadata.js
@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const {validate_signature_auth, get_url_from_req, is_valid_uuid4, get_dir_size, id2path} = require('../helpers');
+const { DB_READ } = require('../services/database/consts');
+
+// -----------------------------------------------------------------------//
+// GET /itemMetadata
+// -----------------------------------------------------------------------//
+router.get('/itemMetadata', async (req, res, next)=>{
+ // Check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // Validate URL signature
+ try{
+ validate_signature_auth(get_url_from_req(req), 'read');
+ }
+ catch(e){
+ console.log(e)
+ return res.status(403).send(e);
+ }
+
+ // Validation
+ if(!req.query.uid)
+ return res.status(400).send('`uid` is required');
+ // uid must be a string
+ else if (req.query.uid && typeof req.query.uid !== 'string')
+ return res.status(400).send('uid must be a string.');
+ // uid cannot be empty
+ else if(req.query.uid && req.query.uid.trim() === '')
+ return res.status(400).send('uid cannot be empty')
+ // uid must be a valid uuid
+ else if(!is_valid_uuid4(req.query.uid))
+ return res.status(400).send('uid must be a valid uuid')
+
+ // modules
+ const {uuid2fsentry} = require('../helpers');
+
+ const uid = req.query.uid;
+
+ const item = await uuid2fsentry(uid);
+
+ // check if item owner is suspended
+ const user = await require('../helpers').get_user({id: item.user_id});
+
+ if (!user) {
+ return res.status(400).send('User not found');
+ }
+
+ if(user.suspended)
+ return res.status(401).send({error: 'Account suspended'});
+
+ if(!item)
+ return res.status(400).send('Item not found');
+
+ const mime = require('mime-types');
+ const contentType = mime.contentType(res.name)
+
+ const itemMetadata = {
+ uid: item.uuid,
+ name: item.name,
+ is_dir: item.is_dir,
+ type: contentType,
+ size: item.is_dir ? await get_dir_size(await id2path(item.id), user) : item.size,
+ created: item.created,
+ modified: item.modified,
+ };
+
+ // ---------------------------------------------------------------//
+ // return_path
+ // ---------------------------------------------------------------//
+ if(req.query.return_path === 'true' || req.query.return_path === '1'){
+ const {id2path} = require('../helpers');
+ itemMetadata.path = await id2path(item.id);
+ }
+ // ---------------------------------------------------------------//
+ // Versions
+ // ---------------------------------------------------------------//
+ if(req.query.return_versions){
+ const db = req.services.get('database').get(DB_READ, 'itemMetadata.js');
+ itemMetadata.versions = [];
+
+ let versions = await db.read(
+ `SELECT * FROM fsentry_versions WHERE fsentry_id = ?`,
+ [item.id]
+ );
+ if(versions.length > 0){
+ for (let index = 0; index < versions.length; index++) {
+ const version = versions[index];
+ itemMetadata.versions.push({
+ id: version.version_id,
+ message: version.message,
+ timestamp: version.ts_epoch,
+ })
+ }
+ }
+ }
+
+ return res.send(itemMetadata)
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/kvstore/clearItems.js b/packages/backend/src/routers/kvstore/clearItems.js
new file mode 100644
index 00000000..6390148a
--- /dev/null
+++ b/packages/backend/src/routers/kvstore/clearItems.js
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../api/APIError");
+const eggspress = require("../../api/eggspress");
+const config = require("../../config");
+
+
+module.exports = eggspress('/clearItems', {
+ subdomain: 'api',
+ auth: true,
+ verified: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+
+ // TODO: model these parameters; validation is contained in brackets
+ // so that it can be easily move.
+ let { app } = req.body;
+
+ // Validation for `app`
+ if ( ! app ) {
+ throw APIError.create('field_missing', null, { key: 'app' });
+ }
+
+ const svc_mysql = req.services.get('mysql');
+ const dbrw = svc_mysql.get(DB_MODE_WRITE, 'kvstore-clearItems');
+ await dbrw.execute(
+ `DELETE FROM kv WHERE user_id=? AND app=?`,
+ [
+ req.user.id,
+ app,
+ ]
+ );
+
+ return res.send({});
+});
diff --git a/packages/backend/src/routers/kvstore/getItem.js b/packages/backend/src/routers/kvstore/getItem.js
new file mode 100644
index 00000000..650a23ea
--- /dev/null
+++ b/packages/backend/src/routers/kvstore/getItem.js
@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../../middleware/auth.js');
+const config = require('../../config.js');
+const { Context } = require('../../util/context.js');
+const { Actor, AppUnderUserActorType, UserActorType } = require('../../services/auth/Actor.js');
+const { DB_READ } = require('../../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /getItem
+// -----------------------------------------------------------------------//
+router.post('/getItem', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../../helpers.js').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // validation
+ if(!req.body.key)
+ return res.status(400).send('`key` is required.');
+ // check size of key, if it's too big then it's an invalid key and we don't want to waste time on it
+ else if(Buffer.byteLength(req.body.key, 'utf8') > config.kv_max_key_size)
+ return res.status(400).send('`key` is too long.');
+
+ const actor = req.body.app
+ ? await Actor.create(AppUnderUserActorType, {
+ user: req.user,
+ app_uid: req.body.app,
+ })
+ : await Actor.create(UserActorType, {
+ user: req.user,
+ })
+ ;
+
+ Context.set('actor', actor);
+
+ // Try KV 1 first
+ const svc_driver = Context.get('services').get('driver');
+ let driver_result;
+ try {
+ const driver_response = await svc_driver.call(
+ 'puter-kvstore', 'get', { key: req.body.key });
+ if ( ! driver_response.success ) {
+ throw new Error(driver_response.error?.message ?? 'Unknown error');
+ }
+ driver_result = driver_response.result;
+ } catch (e) {
+ return res.status(400).send('puter-kvstore driver error: ' + e.message);
+ }
+
+ if ( driver_result ) {
+ return res.send({ key: req.body.key, value: driver_result });
+ }
+
+ // modules
+ const db = req.services.get('database').get(DB_READ, 'getItem-fallback');
+ // get murmurhash module
+ const murmurhash = require('murmurhash')
+ // hash key for faster search in DB
+ const key_hash = murmurhash.v3(req.body.key);
+
+ let kv;
+ // Get value from DB
+ // If app is specified, then get value for that app
+ if(req.body.app){
+ kv = await db.read(
+ `SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1`,
+ [
+ req.user.id,
+ req.body.app,
+ key_hash,
+ ]
+ )
+ // If app is not specified, then get value for global (i.e. system) variables which is app='global'
+ }else{
+ kv = await db.read(
+ `SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global') AND kkey_hash=? LIMIT 1`,
+ [
+ req.user.id,
+ key_hash,
+ ]
+ )
+ }
+
+ // send results to client
+ if(kv[0])
+ return res.send({
+ key: kv[0].kkey,
+ value: kv[0].value,
+ });
+ else
+ return res.send(null)
+})
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/kvstore/listItems.js b/packages/backend/src/routers/kvstore/listItems.js
new file mode 100644
index 00000000..3f101827
--- /dev/null
+++ b/packages/backend/src/routers/kvstore/listItems.js
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../api/APIError");
+const eggspress = require("../../api/eggspress");
+const { DB_READ } = require("../../services/database/consts");
+
+module.exports = eggspress('/listItems', {
+ subdomain: 'api',
+ auth: true,
+ verified: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+
+ let { app } = req.body;
+
+ // Validation for `app`
+ if ( ! app ) {
+ throw APIError.create('field_missing', null, { key: 'app' });
+ }
+
+ const db = req.services.get('database').get(DB_READ, 'kv');
+ let rows = await db.read(
+ `SELECT kkey, value FROM kv WHERE user_id=? AND app=?`,
+ [
+ req.user.id,
+ app,
+ ]
+ );
+
+ rows = rows.map(row => ({
+ key: row.kkey,
+ value: row.value,
+ }));
+
+ return res.send(rows);
+});
diff --git a/packages/backend/src/routers/kvstore/setItem.js b/packages/backend/src/routers/kvstore/setItem.js
new file mode 100644
index 00000000..a3cc3320
--- /dev/null
+++ b/packages/backend/src/routers/kvstore/setItem.js
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../../middleware/auth.js');
+const config = require('../../config.js');
+const {app_exists, byte_format} = require('../../helpers.js');
+const { Actor, AppUnderUserActorType, UserActorType } = require('../../services/auth/Actor.js');
+const { Context } = require('../../util/context.js');
+
+// -----------------------------------------------------------------------//
+// POST /setItem
+// -----------------------------------------------------------------------//
+router.post('/setItem', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../../helpers.js').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+
+ // validation
+ if(!req.body.key)
+ return res.status(400).send('`key` is required');
+ else if(typeof req.body.key !== 'string')
+ return res.status(400).send('`key` must be a string');
+ else if(!req.body.value)
+ return res.status(400).send('`value` is required');
+
+ req.body.key = String(req.body.key);
+ req.body.value = String(req.body.value);
+
+ if(Buffer.byteLength(req.body.key, 'utf8') > config.kv_max_key_size)
+ return res.status(400).send('`key` is too large. Max size is '+byte_format(config.kv_max_key_size)+'.');
+ else if(Buffer.byteLength(req.body.value, 'utf8') > config.kv_max_value_size)
+ return res.status(400).send('`value` is too large. Max size is '+byte_format(config.kv_max_value_size)+'.');
+ else if(req.body.app && !await app_exists({uid: req.body.app}))
+ return res.status(400).send('`app` does not exist');
+
+ // hash key for faster search in DB
+ const murmurhash = require('murmurhash')
+ const key_hash = murmurhash.v3(req.body.key);
+
+ // database connection
+ const db = require('../../db/mysql.js');
+
+ // insert into DB (in case the row isn't migrated yet)
+ await db.promise().execute(
+ `INSERT INTO kv
+ (user_id, app, kkey_hash, kkey, value)
+ VALUES
+ (?, ?, ?, ?, ?)
+ ON DUPLICATE KEY UPDATE
+ value = ?`,
+ [
+ req.user.id,
+ // 0 is reserved for global system variables
+ req.body.app ?? 'global',
+ key_hash,
+ String(req.body.key),
+ String(req.body.value),
+ String(req.body.value),
+ ]
+ )
+
+ // insert into KV 1
+ const actor = req.body.app
+ ? await Actor.create(AppUnderUserActorType, {
+ user: req.user,
+ app_uid: req.body.app,
+ })
+ : await Actor.create(UserActorType, {
+ user: req.user,
+ })
+ ;
+
+ Context.set('actor', actor);
+
+ const svc_driver = Context.get('services').get('driver');
+ let driver_result;
+ try {
+ const driver_response = await svc_driver.call(
+ 'puter-kvstore', 'set', {
+ key: req.body.key,
+ value: req.body.value,
+ });
+ if ( ! driver_response.success ) {
+ throw new Error(driver_response.error?.message ?? 'Unknown error');
+ }
+ driver_result = driver_response.result;
+ } catch (e) {
+ return res.status(400).send('puter-kvstore driver error: ' + e.message);
+ }
+
+ // send results to client
+ return res.send({});
+})
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/login.js b/packages/backend/src/routers/login.js
new file mode 100644
index 00000000..79549790
--- /dev/null
+++ b/packages/backend/src/routers/login.js
@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const { get_user, body_parser_error_handler } = require('../helpers');
+const config = require('../config');
+
+// -----------------------------------------------------------------------//
+// POST /file
+// -----------------------------------------------------------------------//
+router.post('/login', express.json(), body_parser_error_handler, async (req, res, next)=>{
+ // either api. subdomain or no subdomain
+ if(require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '')
+ next();
+
+ // modules
+ const bcrypt = require('bcrypt')
+ const jwt = require('jsonwebtoken')
+ const validator = require('validator')
+
+ // either username or email must be provided
+ if(!req.body.username && !req.body.email)
+ return res.status(400).send('Username or email is required.')
+ // password is required
+ else if(!req.body.password)
+ return res.status(400).send('Password is required.')
+ // password must be a string
+ else if (typeof req.body.password !== 'string' && !(req.body.password instanceof String))
+ return res.status(400).send('Password must be a string.')
+ // if password is too short it's invalid, no need to do a db lookup
+ else if(req.body.password.length < config.min_pass_length)
+ return res.status(400).send('Invalid password.')
+ // username, if present, must be a string
+ else if (req.body.username && typeof req.body.username !== 'string' && !(req.body.username instanceof String))
+ return res.status(400).send('username must be a string.')
+ // if username doesn't pass regex test it's invalid anyway, no need to do DB lookup
+ else if(req.body.username && !req.body.username.match(config.username_regex))
+ return res.status(400).send('Invalid username.')
+ // email, if present, must be a string
+ else if (req.body.email && typeof req.body.email !== 'string' && !(req.body.email instanceof String))
+ return res.status(400).send('email must be a string.')
+ // if email is invalid, no need to do DB lookup anyway
+ else if(req.body.email && !validator.isEmail(req.body.email))
+ return res.status(400).send('Invalid email.')
+
+ // Increment & check rate limit
+ if(kv.incr(`login|${req.ip}|${req.body.email ?? req.body.username}`) > 10)
+ return res.status(429).send('Too many requests.');
+ // Set expiry for rate limit
+ kv.expire(`login|${req.ip}|${req.body.email ?? req.body.username}`, 60*10, 'NX')
+
+ try{
+ let user;
+ // log in using username
+ if(req.body.username){
+ user = await get_user({ username: req.body.username, cached: false });
+ if(!user)
+ return res.status(400).send('Username not found.')
+ }
+ // log in using email
+ else if(validator.isEmail(req.body.email)){
+ user = await get_user({ email: req.body.email, cached: false });
+ if(!user)
+ return res.status(400).send('Email not found.')
+ }
+ // is user suspended?
+ if(user.suspended)
+ return res.status(401).send('This account is suspended.')
+ // pseudo user?
+ // todo make this better, maybe ask them to create an account or send them an activation link
+ if(user.password === null)
+ return res.status(400).send('Incorrect password.')
+ // check password
+ if(await bcrypt.compare(req.body.password, user.password)){
+ const token = await jwt.sign({uuid: user.uuid}, config.jwt_secret)
+ //set cookie
+ // res.cookie(config.cookie_name, token);
+ res.cookie(config.cookie_name, token, {
+ sameSite: 'none',
+ secure: true,
+ httpOnly: true,
+ });
+
+ // send response
+ return res.send({
+ token: token,
+ user:{
+ username: user.username,
+ uuid: user.uuid,
+ email: user.email,
+ email_confirmed: user.email_confirmed,
+ is_temp: (user.password === null && user.email === null),
+ }
+ })
+ }else{
+ return res.status(400).send('Incorrect password.')
+ }
+ }catch(e){
+ return res.status(400).send(e);
+ }
+
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/logout.js b/packages/backend/src/routers/logout.js
new file mode 100644
index 00000000..0e8eddf2
--- /dev/null
+++ b/packages/backend/src/routers/logout.js
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+
+// -----------------------------------------------------------------------//
+// POST /logout
+// -----------------------------------------------------------------------//
+router.post('/logout', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '')
+ next();
+ // delete cookie
+ res.clearCookie(config.cookie_name);
+ //---------------------------------------------------------
+ // DANGER ZONE: delete temp user and all its data
+ //---------------------------------------------------------
+ if(req.user.password === null && req.user.email === null){
+ const { deleteUser } = require('../helpers');
+ deleteUser(req.user.id);
+ }
+ // send response
+ res.send('logged out');
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/open_item.js b/packages/backend/src/routers/open_item.js
new file mode 100644
index 00000000..b4d0599d
--- /dev/null
+++ b/packages/backend/src/routers/open_item.js
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+const eggspress = require('../api/eggspress.js');
+const FSNodeParam = require('../api/filesystem/FSNodeParam.js');
+const { Context } = require('../util/context.js');
+const { UserActorType } = require('../services/auth/Actor.js');
+const APIError = require('../api/APIError.js');
+const { sign_file, suggest_app_for_fsentry, get_app } = require('../helpers.js');
+
+// -----------------------------------------------------------------------//
+// POST /open_item
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/open_item', {
+ subdomain: 'api',
+ auth2: true,
+ verified: true,
+ json: true,
+ allowedMethods: ['POST'],
+ alias: { uid: 'path' },
+ parameters: {
+ subject: new FSNodeParam('path'),
+ }
+}, async (req, res, next) => {
+ const subject = req.values.subject;
+
+ const actor = Context.get('actor');
+ if ( ! actor.type instanceof UserActorType ) {
+ throw APIError.create('forbidden');
+ }
+
+ if ( ! await subject.exists() ) {
+ throw APIError.create('subject_does_not_exist');
+ }
+
+ const svc_acl = Context.get('services').get('acl');
+ if ( ! await svc_acl.check(actor, subject, 'see') ) {
+ throw await svc_acl.get_safe_acl_error(actor, subject, 'see');
+ }
+
+ const signature = await sign_file(subject.entry, 'write');
+ const suggested_apps = await suggest_app_for_fsentry(subject.entry);
+ console.log('suggested apps?', suggested_apps);
+ const apps_only_one = suggested_apps.slice(0,1);
+ const _app = apps_only_one[0];
+ if ( ! _app ) {
+ throw APIError.create('no_suitable_app', null, { entry_name: subject.entry.name });
+ }
+ const app = await get_app(
+ _app.hasOwnProperty('id')
+ ? { id: _app.id }
+ : { uid: _app.uid }
+ ) ?? apps_only_one[0];
+
+ if ( ! app ) {
+ throw APIError.create('no_suitable_app', null, { entry_name: subject.entry.name });
+ }
+
+ // Grant permission to open the file
+ // Note: We always grant write permission here. If the user only
+ // has read permission this is still safe; user permissions
+ // are always checked during an app access.
+ const permission = `fs:${subject.uid}:write`;
+ const svc_permission = Context.get('services').get('permission');
+ await svc_permission.grant_user_app_permission(
+ actor, app.uid, permission, {}, { reason: 'open_item' }
+ );
+
+ // Generate user-app token
+ const svc_auth = Context.get('services').get('auth');
+ const token = await svc_auth.get_user_app_token(app.uid);
+
+ // TODO: DRY
+ // remove some privileged information
+ delete app.id;
+ delete app.approved_for_listing;
+ delete app.approved_for_opening_items;
+ delete app.godmode;
+ delete app.owner_user_id;
+
+ return res.send({
+ signature: signature,
+ token,
+ suggested_apps: [app],
+ });
+});
diff --git a/packages/backend/src/routers/passwd.js b/packages/backend/src/routers/passwd.js
new file mode 100644
index 00000000..450c868f
--- /dev/null
+++ b/packages/backend/src/routers/passwd.js
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const { invalidate_cached_user } = require('../helpers');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+const { DB_WRITE } = require('../services/database/consts');
+
+// -----------------------------------------------------------------------//
+// POST /passwd
+// -----------------------------------------------------------------------//
+router.post('/passwd', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ const db = req.services.get('database').get(DB_WRITE, 'auth');
+ const bcrypt = require('bcrypt');
+
+ if(!req.body.old_pass)
+ return res.status(401).send('old_pass is required')
+ // old_pass must be a string
+ else if (typeof req.body.old_pass !== 'string')
+ return res.status(400).send('old_pass must be a string.')
+ else if(!req.body.new_pass)
+ return res.status(401).send('new_pass is required')
+ // new_pass must be a string
+ else if (typeof req.body.new_pass !== 'string')
+ return res.status(400).send('new_pass must be a string.')
+
+ try{
+ // check old_pass
+ const isMatch = await bcrypt.compare(req.body.old_pass, req.user.password)
+ if(!isMatch)
+ return res.status(400).send('old_pass does not match your current password.')
+ // check new_pass length
+ // todo use config, 6 is hard-coded and wrong
+ else if(req.body.new_pass.length < 6)
+ return res.status(400).send('new_pass must be at least 6 characters long.')
+ else{
+ await db.write(
+ 'UPDATE user SET password=? WHERE `id` = ?',
+ [await bcrypt.hash(req.body.new_pass, 8), req.user.id]
+ );
+ invalidate_cached_user(req.user);
+ return res.send('Password successfully updated.')
+ }
+ }catch(e){
+ return res.status(401).send('an error occured');
+ }
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/query/app.js b/packages/backend/src/routers/query/app.js
new file mode 100644
index 00000000..16857707
--- /dev/null
+++ b/packages/backend/src/routers/query/app.js
@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const eggspress = require("../../api/eggspress");
+const { is_valid_uuid4, get_app } = require("../../helpers");
+const express = require('express');
+const { fuzz_number } = require("../../util/fuzz");
+const kvjs = require("@heyputer/kv.js");
+const { DB_READ } = require("../../services/database/consts");
+
+const PREFIX_APP_UID = 'app-';
+
+module.exports = eggspress('/query/app', {
+ subdomain: 'api',
+ auth: true,
+ verified: true,
+ fs: true,
+ mw: [ express.json({ extended: true }) ],
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ const results = [];
+
+ console.log('BODY?', req.body);
+
+ const db = req.services.get('database').get(DB_READ, 'apps');
+
+ const svc_appInformation = req.services.get('app-information');
+
+ const app_list = [...req.body];
+
+ const collection_fetchers = {
+ recent: async () => {
+ return kvjs.get('apps:recent');
+ }
+ };
+
+ for ( let i=0 ; i < app_list.length ; i++ ) {
+ const P = 'collection:';
+ if ( app_list[i].startsWith(P) ) {
+ let [col_name, amount] = app_list[i].slice(P.length).split(':');
+ if ( amount === undefined ) amount = 20;
+ let uids = svc_appInformation.collections[col_name];
+ uids = uids.slice(0, Math.min(uids.length, amount));
+ console.log('GOT SOME UIDS', uids);
+ app_list.splice(i, 1, ...uids);
+
+ console.log('NEW LIST', app_list);
+ }
+ }
+
+ for ( let i=0 ; i < app_list.length ; i++ ) {
+ const P = 'tag:';
+ if ( app_list[i].startsWith(P) ) {
+ let [tag_name, amount] = app_list[i].slice(P.length).split(':');
+ if ( amount === undefined ) amount = 20;
+ let uids = svc_appInformation.tags[tag_name] ?? [];
+ uids = uids.slice(0, Math.min(uids.length, amount));
+ console.log('GOT SOME UIDS', uids);
+ app_list.splice(i, 1, ...uids);
+
+ console.log('NEW LIST', app_list);
+ }
+ }
+
+ for ( const app_selector_raw of app_list ) {
+ const app_selector =
+ app_selector_raw.startsWith(PREFIX_APP_UID) &&
+ is_valid_uuid4(app_selector_raw.slice(PREFIX_APP_UID.length))
+ ? { uid: app_selector_raw }
+ : { name: app_selector_raw }
+ ;
+
+ const app = await get_app(app_selector);
+ if ( ! app ) continue;
+
+ // uuid, name, title, description, icon, created, filetype_associations, number of users
+
+ // TODO: cache
+ const associations = []; {
+ const res_associations = await db.read(
+ `SELECT * FROM app_filetype_association WHERE app_id = ?`,
+ [app.id]
+ );
+ for ( const row of res_associations ) {
+ associations.push(row.type);
+ }
+ }
+
+ const stats = await svc_appInformation.get_stats(app.uid);
+ for ( const k in stats ) stats[k] = fuzz_number(stats[k]);
+
+ delete stats.open_count;
+
+ // TODO: imply from app model
+ results.push({
+ uuid: app.uid,
+ name: app.name,
+ title: app.title,
+ icon: app.icon,
+ description: app.description,
+ tags: app.tags ? app.tags.split(',') : [],
+ // created: app.timestamp,
+ associations,
+ ...stats,
+ });
+ }
+
+ res.send(results);
+});
diff --git a/packages/backend/src/routers/rao.js b/packages/backend/src/routers/rao.js
new file mode 100644
index 00000000..0dfb8f59
--- /dev/null
+++ b/packages/backend/src/routers/rao.js
@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+// records app opens
+
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+const { is_valid_uuid4, get_app } = require('../helpers');
+const { DB_WRITE } = require('../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /rao
+// -----------------------------------------------------------------------//
+router.post('/rao', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // validation
+ if(!req.body.app_uid || typeof req.body.app_uid !== 'string' && !(req.body.app_uid instanceof String))
+ return res.status(400).send({code: 'invalid_app_uid', message: 'Invalid app uid'});
+ // must be a valid uuid
+ // app uuids start with 'app-', so in order to validate them we remove the prefix first
+ else if(!is_valid_uuid4(req.body.app_uid.replace('app-','')))
+ return res.status(400).send({code: 'invalid_app_uid', message: 'Invalid app uid'});
+
+ // get db connection
+ const db = req.services.get('database').get(DB_WRITE, 'apps');
+
+ // insert into db
+ db.write(
+ `INSERT INTO app_opens (app_uid, user_id, ts) VALUES (?, ?, ?)`,
+ [req.body.app_uid, req.user.id, Math.floor(new Date().getTime() / 1000)]
+ )
+
+ const opened_app = await get_app({uid: req.body.app_uid});
+
+ // -----------------------------------------------------------------------//
+ // Update the 'app opens' cache
+ // -----------------------------------------------------------------------//
+ // First try the cache to see if we have recent apps
+ let recent_apps = kv.get('app_opens:user:' + req.user.id);
+
+ // If cache is not empty, prepend it with the new app
+ if(recent_apps && recent_apps.length > 0){
+ // add the app to the beginning of the array
+ recent_apps.unshift({app_uid: req.body.app_uid});
+
+ // dedupe the array
+ recent_apps = recent_apps.filter((v,i,a)=>a.findIndex(t=>(t.app_uid === v.app_uid))===i);
+
+ // limit to 10
+ recent_apps = recent_apps.slice(0, 10);
+
+ // update cache
+ kv.set('app_opens:user:' + req.user.id, recent_apps);
+ }
+ // Cache is empty, query the db and update the cache
+ else{
+ db.read(
+ 'SELECT DISTINCT app_uid FROM app_opens WHERE user_id = ? GROUP BY app_uid ORDER BY MAX(_id) DESC LIMIT 10',
+ [req.user.id]).then( ([apps]) => {
+ // Update cache with the results from the db (if any results were returned)
+ if(apps && Array.isArray(apps) && apps.length > 0){
+ kv.set('app_opens:user:' + req.user.id, apps);
+ }
+ });
+ }
+
+ // Update clients
+ const socketio = require('../socketio.js').getio();
+ socketio.to(req.user.id).emit('app.opened', {
+ uuid: opened_app.uid,
+ uid: opened_app.uid,
+ name: opened_app.name,
+ title: opened_app.title,
+ icon: opened_app.icon,
+ godmode: opened_app.godmode,
+ maximize_on_start: opened_app.maximize_on_start,
+ index_url: opened_app.index_url,
+ original_client_socket_id: req.body.original_client_socket_id,
+ });
+
+ // return
+ return res.status(200).send({code: 'ok', message: 'ok'});
+});
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/remove-site-dir.js b/packages/backend/src/routers/remove-site-dir.js
new file mode 100644
index 00000000..04ffef1d
--- /dev/null
+++ b/packages/backend/src/routers/remove-site-dir.js
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+
+// -----------------------------------------------------------------------//
+// POST /remove-site-dir
+// -----------------------------------------------------------------------//
+router.post('/remove-site-dir', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // validation
+ if(req.body.dir_uuid === undefined)
+ return res.status(400).send('dir_uuid is required')
+
+ // modules
+ const {uuid2fsentry, chkperm} = require('../helpers');
+ const db = require('../db/mysql.js')
+ const user = req.user
+
+ const item = await uuid2fsentry(req.body.dir_uuid)
+ if(item !== false){
+ // check permission
+ if(!await chkperm(item, req.user.id, 'write'))
+ return res.status(403).send({ code:`forbidden`, message: `permission denied.`})
+ // remove dir/subdomain connection
+ if(req.body.site_uuid)
+ await db.promise().execute(
+ `UPDATE subdomains SET root_dir_id = NULL WHERE user_id = ? AND root_dir_id =? AND uuid = ?`,
+ [user.id, item.id, req.body.site_uuid]
+ );
+ // if site_uuid is undefined, disassociate all websites from this directory
+ else
+ await db.promise().execute(
+ `UPDATE subdomains SET root_dir_id = NULL WHERE user_id = ? AND root_dir_id =?`,
+ [user.id, item.id]
+ );
+
+ res.send({});
+ }else{
+ res.status(400).send();
+ }
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/removeItem.js b/packages/backend/src/routers/removeItem.js
new file mode 100644
index 00000000..430c54ae
--- /dev/null
+++ b/packages/backend/src/routers/removeItem.js
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+
+// -----------------------------------------------------------------------//
+// POST /removeItem
+// -----------------------------------------------------------------------//
+router.post('/removeItem', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // validation
+ if(!req.body.key)
+ return res.status(400).send('`key` is required');
+ // check size of key, if it's too big then it's an invalid key and we don't want to waste time on it
+ else if(Buffer.byteLength(req.body.key, 'utf8') > config.kv_max_key_size)
+ return res.status(400).send('`key` is too long.');
+ else if(!req.body.app)
+ return res.status(400).send('`app` is required');
+
+ // modules
+ const db = require('../db/mysql.js');
+ // get murmurhash module
+ const murmurhash = require('murmurhash')
+ // hash key for faster search in DB
+ const key_hash = murmurhash.v3(req.body.key);
+
+ // insert into DB
+ let [kv] = await db.promise().execute(
+ `DELETE FROM kv WHERE user_id=? AND app = ? AND kkey_hash = ? LIMIT 1`,
+ [
+ req.user.id,
+ req.body.app ?? 'global',
+ key_hash,
+ ]
+ )
+
+ // send results to client
+ return res.send({});
+})
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/save_account.js b/packages/backend/src/routers/save_account.js
new file mode 100644
index 00000000..da3a3955
--- /dev/null
+++ b/packages/backend/src/routers/save_account.js
@@ -0,0 +1,196 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const {get_taskbar_items, username_exists, send_email_verification_code, send_email_verification_token, invalidate_cached_user} = require('../helpers');
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+const { Context } = require('../util/context');
+const { DB_WRITE } = require('../services/database/consts');
+
+// -----------------------------------------------------------------------//
+// POST /save_account
+// -----------------------------------------------------------------------//
+router.post('/save_account', auth, express.json(), async (req, res, next)=>{
+ // either api. subdomain or no subdomain
+ if(require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '')
+ next();
+
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'auth');
+ const validator = require('validator')
+ const bcrypt = require('bcrypt')
+ const jwt = require('jsonwebtoken')
+ const { v4: uuidv4 } = require('uuid');
+
+ // validation
+ if(req.user.password !== null)
+ return res.status(400).send('User account already saved.');
+ else if(!req.body.username)
+ return res.status(400).send('Username is required');
+ // username must be a string
+ else if (typeof req.body.username !== 'string')
+ return res.status(400).send('username must be a string.')
+ else if(!req.body.username.match(config.username_regex))
+ return res.status(400).send('Username can only contain letters, numbers and underscore (_).')
+ else if(req.body.username.length > config.username_max_length)
+ return res.status(400).send(`Username cannot have more than ${config.username_max_length} characters.`)
+ // check if username matches any reserved words
+ else if(config.reserved_words.includes(req.body.username))
+ return res.status(400).send({message: 'This username is not available.'});
+ else if(!req.body.email)
+ return res.status(400).send('Email is required')
+ // email must be a string
+ else if (typeof req.body.email !== 'string')
+ return res.status(400).send('email must be a string.')
+ else if(!validator.isEmail(req.body.email))
+ return res.status(400).send('Please enter a valid email address.')
+ else if(!req.body.password)
+ return res.status(400).send('Password is required')
+ // password must be a string
+ else if (typeof req.body.password !== 'string')
+ return res.status(400).send('password must be a string.')
+ else if(req.body.password.length < config.min_pass_length)
+ return res.status(400).send(`Password must be at least ${config.min_pass_length} characters long.`)
+
+ // duplicate username check, do this only if user has supplied a new username
+ if(req.body.username !== req.user.username && await username_exists(req.body.username))
+ return res.status(400).send('This username already exists in our database. Please use another one.');
+ // duplicate email check (pseudo-users don't count)
+ let rows2 = await db.read(`SELECT EXISTS(SELECT 1 FROM user WHERE email=? AND password IS NOT NULL) AS email_exists`, [req.body.email]);
+ if(rows2[0].email_exists)
+ return res.status(400).send('This email already exists in our database. Please use another one.');
+ // get pseudo user, if exists
+ let pseudo_user = await db.read(`SELECT * FROM user WHERE email = ? AND password IS NULL`, [req.body.email]);
+ pseudo_user = pseudo_user[0];
+ // get uuid user, if exists
+ if(req.body.uuid){
+ uuid_user = await db.read(`SELECT * FROM user WHERE uuid = ? LIMIT 1`, [req.body.uuid]);
+ uuid_user = uuid_user[0];
+ }
+
+ // send_confirmation_code
+ req.body.send_confirmation_code = req.body.send_confirmation_code ?? true;
+
+ // todo email confirmation is required by default unless:
+ // Pseudo user converting and matching uuid is provided
+ let email_confirmation_required = 0;
+
+ // -----------------------------------
+ // Get referral user
+ // -----------------------------------
+ let referred_by_user = undefined;
+ if ( req.body.referral_code ) {
+ referred_by_user = await get_user({ referral_code: req.body.referral_code });
+ if ( ! referred_by_user ) {
+ return res.status(400).send('Referral code not found');
+ }
+ }
+
+ // -----------------------------------
+ // New User
+ // -----------------------------------
+ const user_uuid = req.user.uuid;
+ let email_confirm_code = Math.floor(100000 + Math.random() * 900000);
+ const email_confirm_token = uuidv4();
+
+ if(pseudo_user === undefined){
+ await db.write(
+ `UPDATE user
+ SET
+ username = ?, email = ?, password = ?, email_confirm_code = ?, email_confirm_token = ?${
+ referred_by_user ? ', referred_by = ?' : '' }
+ WHERE
+ id = ?`,
+ [
+ // username
+ req.body.username,
+ // email
+ req.body.email,
+ // password
+ await bcrypt.hash(req.body.password, 8),
+ // email_confirm_code
+ email_confirm_code,
+ //email_confirm_token
+ email_confirm_token,
+ // referred_by
+ ...(referred_by_user ? [referred_by_user.id] : []),
+ // id
+ req.user.id
+ ]
+ );
+ invalidate_cached_user(req.user);
+
+ // Update root directory name
+ await db.write(
+ `UPDATE fsentries SET name = ? WHERE user_id = ? and parent_uid IS NULL`,
+ [
+ // name
+ req.body.username,
+ // id
+ req.user.id,
+ ]
+ );
+ const filesystem = req.services.get('filesystem');
+ await filesystem.update_child_paths(`/${req.user.username}`, `/${req.body.username}`, req.user.id);
+
+ if(req.body.send_confirmation_code)
+ send_email_verification_code(email_confirm_code, req.body.email);
+ else
+ send_email_verification_token(email_confirm_token, req.body.email, user_uuid);
+ }
+
+ // create token for login
+ const token = await jwt.sign({uuid: user_uuid}, config.jwt_secret);
+
+ // user id
+ // todo if pseudo user, assign directly no need to do another DB lookup
+ const user_id = req.user.id;
+ const user_res = await db.read('SELECT * FROM `user` WHERE `id` = ? LIMIT 1', [user_id]);
+ const user = user_res[0];
+
+ // todo send LINK-based verification email
+
+ //set cookie
+ res.cookie(config.cookie_name, token);
+
+ {
+ const svc_event = req.services.get('event');
+ svc_event.emit('user.save_account', { user });
+ }
+
+ // return results
+ return res.send({
+ token: token,
+ user:{
+ username: user.username,
+ uuid: user.uuid,
+ email: user.email,
+ is_temp: false,
+ requires_email_confirmation: user.requires_email_confirmation,
+ email_confirmed: user.email_confirmed,
+ email_confirmation_required: email_confirmation_required,
+ taskbar_items: await get_taskbar_items(user),
+ referral_code: user.referral_code,
+ }
+ })
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/send-confirm-email.js b/packages/backend/src/routers/send-confirm-email.js
new file mode 100644
index 00000000..974b7884
--- /dev/null
+++ b/packages/backend/src/routers/send-confirm-email.js
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+const {send_email_verification_code, invalidate_cached_user} = require('../helpers');
+const { DB_WRITE } = require('../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /send-confirm-email
+// -----------------------------------------------------------------------//
+router.post('/send-confirm-email', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ const db = req.services.get('database').get(DB_WRITE, 'auth');
+ let email_confirm_code = Math.floor(100000 + Math.random() * 900000);
+
+ if(req.user.suspended)
+ return res.status(401).send({error: 'Account suspended'});
+
+ await db.write(
+ `UPDATE user SET email_confirm_code = ? WHERE id = ?`,
+ [
+ // email_confirm_code
+ email_confirm_code,
+ // id
+ req.user.id,
+ ]);
+ invalidate_cached_user(req.user);
+
+ // send email verification
+ send_email_verification_code(email_confirm_code, req.user.email);
+
+ res.send();
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/send-pass-recovery-email.js b/packages/backend/src/routers/send-pass-recovery-email.js
new file mode 100644
index 00000000..c078ddde
--- /dev/null
+++ b/packages/backend/src/routers/send-pass-recovery-email.js
@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const { body_parser_error_handler, get_user, invalidate_cached_user } = require('../helpers');
+const config = require('../config');
+const { DB_WRITE } = require('../services/database/consts');
+
+// -----------------------------------------------------------------------//
+// POST /send-pass-recovery-email
+// -----------------------------------------------------------------------//
+router.post('/send-pass-recovery-email', express.json(), body_parser_error_handler, async (req, res, next)=>{
+ // either api. subdomain or no subdomain
+ if(require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '')
+ next();
+
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'auth');
+ const validator = require('validator')
+
+ // validation
+ if(!req.body.username && !req.body.email)
+ return res.status(400).send('Username or email is required.');
+ // username, if provided, must be a string
+ else if (req.body.username && typeof req.body.username !== 'string')
+ return res.status(400).send('username must be a string.')
+ // if username doesn't pass regex test it's invalid anyway, no need to do DB lookup
+ else if(req.body.username && !req.body.username.match(config.username_regex))
+ return res.status(400).send('Invalid username.')
+ // email, if provided, must be a string
+ else if (req.body.email && typeof req.body.email !== 'string')
+ return res.status(400).send('email must be a string.')
+ // if email is invalid, no need to do DB lookup anyway
+ else if(req.body.email && !validator.isEmail(req.body.email))
+ return res.status(400).send('Invalid email.')
+
+ try{
+ let user;
+ // see if username exists
+ if(req.body.username){
+ user = await get_user({username: req.body.username});
+ if(!user)
+ return res.status(400).send('Username not found.')
+ }
+ // see if email exists
+ else if(req.body.email){
+ user = await get_user({email: req.body.email});
+ if(!user)
+ return res.status(400).send('Email not found.')
+ }
+
+ // check if user is suspended
+ if(user.suspended){
+ return res.status(401).send('Account suspended');
+ }
+ // set pass_recovery_token
+ const { v4: uuidv4 } = require('uuid');
+ const nodemailer = require("nodemailer");
+ const token = uuidv4();
+ await db.write(
+ 'UPDATE user SET pass_recovery_token=? WHERE `id` = ?',
+ [token, user.id]
+ );
+ invalidate_cached_user(user);
+
+ // prepare email
+ let transporter = nodemailer.createTransport({
+ host: config.smtp_server,
+ port: config.smpt_port,
+ secure: true, // STARTTLS
+ auth: {
+ user: config.smtp_username,
+ pass: config.smtp_password,
+ },
+ });
+
+ // create link
+ const rec_link = config.origin + '/action/set-new-password?user=' + user.uuid + '&token=' + token;
+
+ // send email
+ transporter.sendMail({
+ from: 'no-reply@puter.com', // sender address
+ to: user.email, // list of receivers
+ subject: "Password Recovery", // Subject line
+ html: `
+ Hi there,
+ A password recovery request was issued for your account, please follow the link below to reset your password:
+ ${rec_link}
+ Sincerely,
+ Puter
`,
+ });
+
+ // Send response
+ if(req.body.username)
+ return res.send({message: `Password recovery sent to the email associated with ${user.username} . Please check your email for instructions on how to reset your password.`});
+ else
+ return res.send({message: `Password recovery email sent to ${user.email} . Please check your email for instructions on how to reset your password.`});
+
+ }catch(e){
+ console.log(e)
+ return res.status(400).send(e);
+ }
+
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/set-desktop-bg.js b/packages/backend/src/routers/set-desktop-bg.js
new file mode 100644
index 00000000..a23e54cc
--- /dev/null
+++ b/packages/backend/src/routers/set-desktop-bg.js
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const config = require('../config.js');
+const { invalidate_cached_user } = require('../helpers');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+const { DB_WRITE } = require('../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /set-desktop-bg
+// -----------------------------------------------------------------------//
+router.post('/set-desktop-bg', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'ui');
+
+ // insert into DB
+ await db.write(
+ `UPDATE user SET desktop_bg_url = ?, desktop_bg_color = ?, desktop_bg_fit = ? WHERE user.id = ?`,
+ [
+ req.body.url ?? null,
+ req.body.color ?? null,
+ req.body.fit ?? null,
+ req.user.id,
+ ]
+ )
+ invalidate_cached_user(req.user);
+
+ // send results to client
+ return res.send({});
+})
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/set-pass-using-token.js b/packages/backend/src/routers/set-pass-using-token.js
new file mode 100644
index 00000000..4f3ab24e
--- /dev/null
+++ b/packages/backend/src/routers/set-pass-using-token.js
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express')
+const router = new express.Router()
+const config = require('../config')
+const { invalidate_cached_user_by_id } = require('../helpers')
+const { DB_WRITE } = require('../services/database/consts')
+
+// -----------------------------------------------------------------------//
+// POST /set-pass-using-token
+// -----------------------------------------------------------------------//
+router.post('/set-pass-using-token', express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '')
+ next();
+
+ // modules
+ const bcrypt = require('bcrypt');
+
+ const db = req.services.get('database').get(DB_WRITE, 'auth');
+
+ // password is required
+ if(!req.body.password)
+ return res.status(401).send('password is required')
+ // user_id is required
+ else if(!req.body.user_id)
+ return res.status(401).send('user_id is required')
+ // token is required
+ else if(!req.body.token)
+ return res.status(401).send('token is required')
+ // password must be a string
+ else if(typeof req.body.password !== 'string')
+ return res.status(400).send('password must be a string.')
+ // check password length
+ else if(req.body.password.length < config.min_pass_length)
+ return res.status(400).send(`Password must be at least ${config.min_pass_length} characters long.`)
+
+ try{
+ await db.write(
+ 'UPDATE user SET password=?, pass_recovery_token=NULL WHERE `uuid` = ? AND pass_recovery_token = ?',
+ [await bcrypt.hash(req.body.password, 8), req.body.user_id, req.body.token]
+ );
+ invalidate_cached_user_by_id(req.body.user_id);
+
+ return res.send('Password successfully updated.')
+ }catch(e){
+ return res.status(500).send('An internal error occured.');
+ }
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/set_layout.js b/packages/backend/src/routers/set_layout.js
new file mode 100644
index 00000000..75c4eefb
--- /dev/null
+++ b/packages/backend/src/routers/set_layout.js
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+const { DB_WRITE } = require('../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /set_layout
+// -----------------------------------------------------------------------//
+router.post('/set_layout', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // validation
+ if(req.body.item_uid === undefined && req.body.item_path === undefined)
+ return res.status(400).send('`item_uid` or `item_path` is required');
+ else if(req.body.layout === undefined)
+ return res.status(400).send('`layout` is required');
+ else if(req.body.layout !== 'icons' && req.body.layout !== 'details' && req.body.layout !== 'list')
+ return res.status(400).send('invalid `layout`');
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'ui');
+ const {uuid2fsentry, convert_path_to_fsentry, chkperm} = require('../helpers');
+
+ //get dir
+ let item;
+ if(req.body.item_uid)
+ item = await uuid2fsentry(req.body.item_uid);
+ else if(req.body.item_path)
+ item = await convert_path_to_fsentry(req.body.item_path);
+
+ // item not found
+ if(item === false){
+ return res.status(400).send({
+ error:{
+ message: 'No entry found with this uid'
+ }
+ })
+ }
+
+ // must be dir
+ if(!item.is_dir)
+ return res.status(400).send(`must be a directory`);
+
+ // check permission
+ if(!await chkperm(item, req.user.id, 'write'))
+ return res.status(403).send({ code:`forbidden`, message: `permission denied.`})
+
+ // insert into DB
+ await db.write(
+ `UPDATE fsentries SET layout = ? WHERE id = ?`,
+ [req.body.layout, item.id]
+ )
+
+ // send results to client
+ return res.send({});
+})
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/set_sort_by.js b/packages/backend/src/routers/set_sort_by.js
new file mode 100644
index 00000000..2ba54756
--- /dev/null
+++ b/packages/backend/src/routers/set_sort_by.js
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+const { DB_WRITE } = require('../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /set_sort_by
+// -----------------------------------------------------------------------//
+router.post('/set_sort_by', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // validation
+ if(req.body.item_uid === undefined && req.body.item_path === undefined)
+ return res.status(400).send('`item_uid` or `item_path` is required');
+ else if(req.body.sort_by === undefined)
+ return res.status(400).send('`sort_by` is required');
+ else if(req.body.sort_by !== 'name' && req.body.sort_by !== 'size' && req.body.sort_by !== 'modified' && req.body.sort_by !== 'type')
+ return res.status(400).send('invalid `sort_by`');
+ else if(req.body.sort_order !== 'asc' && req.body.sort_order !== 'desc')
+ return res.status(400).send('invalid `sort_order`');
+
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'ui');
+ const {uuid2fsentry, convert_path_to_fsentry, chkperm} = require('../helpers');
+
+ //get dir
+ let item;
+ if(req.body.item_uid)
+ item = await uuid2fsentry(req.body.item_uid);
+ else if(req.body.item_path)
+ item = await convert_path_to_fsentry(req.body.item_path);
+
+ // item not found
+ if(item === false){
+ return res.status(400).send({
+ error:{
+ message: 'No entry found with this uid'
+ }
+ })
+ }
+
+ // must be dir
+ if(!item.is_dir)
+ return res.status(400).send(`must be a directory`);
+
+ // check permission
+ if(!await chkperm(item, req.user.id, 'write'))
+ return res.status(403).send({ code:`forbidden`, message: `permission denied.`})
+
+ // set sort_by
+ await db.write(
+ `UPDATE fsentries SET sort_by = ? WHERE id = ?`,
+ [req.body.sort_by, item.id]
+ )
+
+ // set sort_order
+ await db.write(
+ `UPDATE fsentries SET sort_order = ? WHERE id = ?`,
+ [req.body.sort_order, item.id]
+ )
+
+ // send results to client
+ return res.send({});
+})
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/sign.js b/packages/backend/src/routers/sign.js
new file mode 100644
index 00000000..9a8d6aa5
--- /dev/null
+++ b/packages/backend/src/routers/sign.js
@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const {sign_file, convert_path_to_fsentry, uuid2fsentry, chkperm, get_app} = require('../helpers');
+const eggspress = require('../api/eggspress.js');
+const APIError = require('../api/APIError.js');
+const { Context } = require('../util/context.js');
+const { UserActorType } = require('../services/auth/Actor.js');
+
+// -----------------------------------------------------------------------//
+// POST /sign
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/sign', {
+ subdomain: 'api',
+ auth2: true,
+ verified: true,
+ json: true,
+ allowedMethods: ['POST'],
+}, async (req, res, next)=>{
+ const actor = Context.get('actor');
+ if ( ! actor.type instanceof UserActorType ) {
+ throw APIError.create('forbidden');
+ }
+
+ if(!req.body.items) {
+ throw APIError.create('field_missing', null, { key: 'items' });
+ }
+
+ let items = Array.isArray(req.body.items) ? req.body.items : [res];
+ let signatures = [];
+
+ const svc_fs = Context.get('services').get('filesystem');
+
+ const result = {
+ signatures
+ };
+
+ let app = null;
+ if ( req.body.app_uid ) {
+ if ( typeof req.body.app_uid !== 'string' ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'app_uid',
+ expected: 'string'
+ });
+ }
+
+ app = await get_app({ uid: req.body.app_uid });
+ if ( ! app ) {
+ throw APIError.create('no_suitable_app', null, { entry_name: subject.entry.name });
+ }
+ // Generate user-app token
+ const svc_auth = Context.get('services').get('auth');
+ const token = await svc_auth.get_user_app_token(app.uid);
+ result.token = token;
+ }
+
+ for (let index = 0; index < items.length; index++) {
+ let item = items[index];
+ if ( ! item ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'items',
+ expected: 'each item to have: (uid OR path) AND action'
+ }).serialize()
+ }
+
+ if ( typeof item !== 'object' || Array.isArray(item) ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'items',
+ expected: 'each item to be an object'
+ }).serialize()
+ }
+
+ // validation
+ if((!item.uid && !item.path)|| !item.action){
+ throw APIError.create('field_invalid', null, {
+ key: 'items',
+ expected: 'each item to have: (uid OR path) AND action'
+ }).serialize()
+ }
+
+ if ( typeof item.uid !== 'string' && typeof item.path !== 'string' ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'items',
+ expected: 'each item to have only string values for uid and path'
+ }).serialize()
+ }
+
+ const node = await svc_fs.node(item);
+
+ if ( ! await node.exists() ) {
+ // throw APIError.create('subject_does_not_exist').serialize()
+ signatures.push({})
+ continue;
+ }
+
+ const svc_acl = Context.get('services').get('acl');
+ if ( ! await svc_acl.check(actor, node, 'see') ) {
+ throw await svc_acl.get_safe_acl_error(actor, subject, 'see');
+ }
+
+ if ( app !== null ) {
+ // Grant write permission to app
+ const svc_permission = Context.get('services').get('permission');
+ const permission = `fs:${await node.get('uid')}:write`;
+ await svc_permission.grant_user_app_permission(
+ actor, app.uid, permission, {}, { reason: 'endpoint:sign' }
+ );
+ }
+
+ // sign
+ try{
+ let signature = await sign_file(node.entry, item.action)
+ signature.path = signature.path ?? item.path;
+ signatures.push(signature);
+ }
+ catch(e){
+ signatures.push({})
+ }
+ }
+
+
+ res.send(result);
+})
diff --git a/packages/backend/src/routers/signup.js b/packages/backend/src/routers/signup.js
new file mode 100644
index 00000000..ca45d9d7
--- /dev/null
+++ b/packages/backend/src/routers/signup.js
@@ -0,0 +1,298 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const {get_taskbar_items, generate_random_username, generate_system_fsentries, body_parser_error_handler, send_email_verification_code, send_email_verification_token, username_exists, invalidate_cached_user_by_id, get_user } = require('../helpers');
+const config = require('../config');
+const eggspress = require('../api/eggspress');
+const { Context } = require('../util/context');
+const { DB_WRITE } = require('../services/database/consts');
+
+// -----------------------------------------------------------------------//
+// POST /signup
+// -----------------------------------------------------------------------//
+module.exports = eggspress(['/signup'], {
+ allowedMethods: ['POST'],
+ alarm_timeout: 7000, // when it calls us
+ response_timeout: 20000, // when it gives up
+ abuse: {
+ no_bots: true,
+ // puter_origin: false,
+ shadow_ban_responder: (req, res) => {
+ res.status(400).send(`email username mismatch; please provide a password`);
+ }
+ },
+}, async (req, res, next) => {
+ // either api. subdomain or no subdomain
+ if(require('../helpers').subdomain(req) !== 'api' && require('../helpers').subdomain(req) !== '')
+ next();
+
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'auth');
+ const bcrypt = require('bcrypt')
+ const { v4: uuidv4 } = require('uuid');
+ const jwt = require('jsonwebtoken')
+ const validator = require('validator')
+ let uuid_user;
+
+ const svc_authAudit = Context.get('services').get('auth-audit');
+ svc_authAudit.record({
+ requester: Context.get('requester'),
+ action: req.body.is_temp ? `signup:temp` : `signup:real`,
+ body: req.body,
+ });
+
+ // check bot trap, if `p102xyzname` is anything but an empty string it means
+ // that a bot has filled the form
+ // doesn't apply to temp users
+ if(!req.body.is_temp && req.body.p102xyzname !== '')
+ return res.send();
+
+ // check if user is already logged in
+ if ( req.body.is_temp && req.cookies[config.cookie_name] ) {
+ const token = req.cookies[config.cookie_name];
+ const decoded = await jwt.verify(token, config.jwt_secret);
+ const user = await get_user({ uuid: decoded.uuid });
+ if ( user ) {
+ return res.send({
+ token: token,
+ user: {
+ username: user.username,
+ uuid: user.uuid,
+ email: user.email,
+ email_confirmed: user.email_confirmed,
+ requires_email_confirmation: user.requires_email_confirmation,
+ is_temp: (user.password === null && user.email === null),
+ taskbar_items: await get_taskbar_items(user),
+ }
+ });
+ }
+ }
+
+ // temporary user
+ if(req.body.is_temp){
+ req.body.username = await generate_random_username();
+ req.body.email = req.body.username + '@gmail.com';
+ req.body.password = 'sadasdfasdfsadfsa';
+ }
+
+ // send_confirmation_code
+ req.body.send_confirmation_code = req.body.send_confirmation_code ?? true;
+
+ // username is required
+ if(!req.body.username)
+ return res.status(400).send('Username is required')
+ // username must be a string
+ else if (typeof req.body.username !== 'string')
+ return res.status(400).send('username must be a string.')
+ // check if username is valid
+ else if(!req.body.username.match(config.username_regex))
+ return res.status(400).send('Username can only contain letters, numbers and underscore (_).')
+ // check if username is of proper length
+ else if(req.body.username.length > config.username_max_length)
+ return res.status(400).send(`Username cannot be longer than ${config.username_max_length} characters.`)
+ // check if username matches any reserved words
+ else if(config.reserved_words.includes(req.body.username))
+ return res.status(400).send({message: 'This username is not available.'});
+ // TODO: DRY: change_email.js
+ else if(!req.body.is_temp && !req.body.email)
+ return res.status(400).send('Email is required');
+ // email, if present, must be a string
+ else if (req.body.email && typeof req.body.email !== 'string')
+ return res.status(400).send('email must be a string.')
+ // if email is present, validate it
+ else if(!req.body.is_temp && !validator.isEmail(req.body.email))
+ return res.status(400).send('Please enter a valid email address.')
+ else if(!req.body.is_temp && !req.body.password)
+ return res.status(400).send('Password is required');
+ // password, if present, must be a string
+ else if (req.body.password && typeof req.body.password !== 'string')
+ return res.status(400).send('password must be a string.')
+ else if(!req.body.is_temp && req.body.password.length < config.min_pass_length)
+ return res.status(400).send(`Password must be at least ${config.min_pass_length} characters long.`);
+
+ // duplicate username check
+ if(await username_exists(req.body.username))
+ return res.status(400).send('This username already exists in our database. Please use another one.');
+ // duplicate email check (pseudo-users don't count)
+ let rows2 = await db.read(`SELECT EXISTS(SELECT 1 FROM user WHERE email=? AND password IS NOT NULL) AS email_exists`, [req.body.email]);
+ if(rows2[0].email_exists)
+ return res.status(400).send('This email already exists in our database. Please use another one.');
+ // get pseudo user, if exists
+ let pseudo_user = await db.read(`SELECT * FROM user WHERE email = ? AND password IS NULL`, [req.body.email]);
+ pseudo_user = pseudo_user[0];
+ // get uuid user, if exists
+ if(req.body.uuid){
+ uuid_user = await db.read(`SELECT * FROM user WHERE uuid = ? LIMIT 1`, [req.body.uuid]);
+ uuid_user = uuid_user[0];
+ }
+
+ // email confirmation is required by default unless:
+ // Pseudo user converting and matching uuid is provided
+ let email_confirmation_required = 1;
+ if(pseudo_user && uuid_user && pseudo_user.id === uuid_user.id)
+ email_confirmation_required = 0;
+
+ // -----------------------------------
+ // Get referral user
+ // -----------------------------------
+ let referred_by_user = undefined;
+ if ( req.body.referral_code ) {
+ referred_by_user = await get_user({ referral_code: req.body.referral_code });
+ if ( ! referred_by_user ) {
+ return res.status(400).send('Referral code not found');
+ }
+ }
+
+ // -----------------------------------
+ // New User
+ // -----------------------------------
+ const user_uuid = uuidv4();
+ const email_confirm_token = uuidv4();
+ let insert_res;
+ let email_confirm_code = Math.floor(100000 + Math.random() * 900000);
+
+ if(pseudo_user === undefined){
+ insert_res = await db.write(
+ `INSERT INTO user
+ (username, email, password, uuid, referrer, email_confirm_code, email_confirm_token, free_storage, referred_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ // username
+ req.body.username,
+ // email
+ req.body.is_temp ? null : req.body.email,
+ // password
+ req.body.is_temp ? null : await bcrypt.hash(req.body.password, 8),
+ // uuid
+ user_uuid,
+ // referrer
+ req.body.referrer ?? null,
+ // email_confirm_code
+ email_confirm_code,
+ // email_confirm_token
+ email_confirm_token,
+ // free_storage
+ config.storage_capacity,
+ // referred_by
+ referred_by_user ? referred_by_user.id : null,
+ ]);
+
+ // record activity
+ db.write(
+ 'UPDATE `user` SET `last_activity_ts` = now() WHERE id=? LIMIT 1',
+ [insert_res.insertId]
+ );
+ }
+ // -----------------------------------
+ // Pseudo User converting
+ // -----------------------------------
+ else{
+ insert_res = await db.write(
+ `UPDATE user SET
+ username = ?, password = ?, uuid = ?, email_confirm_code = ?, email_confirm_token = ?, email_confirmed = ?, requires_email_confirmation = 1,
+ referred_by = ?
+ WHERE id = ?`,
+ [
+ // username
+ req.body.username,
+ // password
+ await bcrypt.hash(req.body.password, 8),
+ // uuid
+ user_uuid,
+ // email_confirm_code
+ email_confirm_code,
+ // email_confirm_token
+ email_confirm_token,
+ // email_confirmed
+ !email_confirmation_required,
+ // id
+ pseudo_user.id,
+ // referred_by
+ referred_by_user ? referred_by_user.id : null,
+ ]
+ );
+
+ // record activity
+ db.write('UPDATE `user` SET `last_activity_ts` = now() WHERE id=? LIMIT 1', [pseudo_user.id]);
+ invalidate_cached_user_by_id(pseudo_user.id);
+ }
+ // create token for login
+ const token = await jwt.sign({uuid: user_uuid}, config.jwt_secret);
+
+ // user id
+ // todo if pseudo user, assign directly no need to do another DB lookup
+ const user_id = (pseudo_user === undefined) ? insert_res.insertId : pseudo_user.id;
+ const [user] = await db.read(
+ 'SELECT * FROM `user` WHERE `id` = ? LIMIT 1',
+ [user_id]
+ );
+
+ //-------------------------------------------------------------
+ // email confirmation
+ //-------------------------------------------------------------
+ if((!req.body.is_temp && email_confirmation_required) || user.requires_email_confirmation){
+ if(req.body.send_confirmation_code || user.requires_email_confirmation)
+ send_email_verification_code(email_confirm_code, user.email);
+ else
+ send_email_verification_token(user.email_confirm_token, user.email, user.uuid);
+ }
+
+ //-------------------------------------------------------------
+ // referral code
+ //-------------------------------------------------------------
+ let referral_code;
+ if ( pseudo_user === undefined ) {
+ const svc_referralCode = Context.get('services')
+ .get('referral-code', { optional: true });
+ if ( svc_referralCode ) {
+ referral_code = await svc_referralCode.gen_referral_code(user);
+ }
+ }
+
+ await generate_system_fsentries(user);
+
+ //set cookie
+ res.cookie(config.cookie_name, token, {
+ sameSite: 'none',
+ secure: true,
+ httpOnly: true,
+ });
+
+ // add to mailchimp
+ if(!req.body.is_temp){
+ const svc_event = Context.get('services').get('event');
+ svc_event.emit('user.save_account', { user });
+ }
+
+ // return results
+ return res.send({
+ token: token,
+ user:{
+ username: user.username,
+ uuid: user.uuid,
+ email: user.email,
+ email_confirmed: user.email_confirmed,
+ requires_email_confirmation: user.requires_email_confirmation,
+ is_temp: (user.password === null && user.email === null),
+ taskbar_items: await get_taskbar_items(user),
+ referral_code,
+ }
+ })
+});
diff --git a/packages/backend/src/routers/sites.js b/packages/backend/src/routers/sites.js
new file mode 100644
index 00000000..df4d5c4d
--- /dev/null
+++ b/packages/backend/src/routers/sites.js
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+
+// -----------------------------------------------------------------------//
+// POST /sites
+// -----------------------------------------------------------------------//
+router.post('/sites', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // modules
+ const {id2path} = require('../helpers');
+ let db = require('../db/mysql.js')
+ let dbrr = db.readReplica ?? db;
+
+ const user = req.user
+ const sites = [];
+
+ let [subdomains] = await dbrr.promise().execute(
+ `SELECT * FROM subdomains WHERE user_id = ?`,
+ [user.id]
+ );
+ if(subdomains.length > 0){
+ for(let i=0; i< subdomains.length; i++){
+ let site = {};
+ // address
+ site.address = config.protocol + '://' + subdomains[i].subdomain + '.' + 'puter.site';
+ // uuid
+ site.uuid = subdomains[i].uuid;
+ // dir
+ let [dir] = await dbrr.promise().execute(
+ `SELECT * FROM fsentries WHERE id = ?`,
+ [subdomains[i].root_dir_id]
+ );
+
+ if(dir.length > 0){
+ site.has_dir = true;
+ site.dir_uid = dir[0].uuid;
+ site.dir_name = dir[0].name;
+ site.dir_path = await id2path(dir[0].id)
+ }else{
+ site.has_dir = false;
+ }
+
+ sites.push(site);
+ }
+ }
+ res.send(sites);
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/suggest_apps.js b/packages/backend/src/routers/suggest_apps.js
new file mode 100644
index 00000000..4982a4e7
--- /dev/null
+++ b/packages/backend/src/routers/suggest_apps.js
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+const config = require('../config');
+
+// -----------------------------------------------------------------------//
+// POST /suggest_apps
+// -----------------------------------------------------------------------//
+router.post('/suggest_apps', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // validation
+ if(req.body.uid === undefined && req.body.path === undefined)
+ return res.status(400).send({message: '`uid` or `path` required'})
+
+ // modules
+ const {convert_path_to_fsentry, uuid2fsentry, chkperm, suggest_app_for_fsentry} = require('../helpers')
+ let fsentry;
+
+ // by uid
+ if(req.body.uid)
+ fsentry = await uuid2fsentry(req.body.uid);
+ // by path
+ else{
+ fsentry = await convert_path_to_fsentry(req.body.path);
+ if(fsentry === false)
+ return res.status(400).send('Path not found.');
+ }
+
+ // check permission
+ if(!await chkperm(fsentry, req.user.id, 'read'))
+ return res.status(403).send({ code:`forbidden`, message: `permission denied.`});
+
+ // get suggestions
+ try{
+ return res.send(await suggest_app_for_fsentry(fsentry));
+ }
+ catch(e){
+ return res.status(400).send(e)
+ }
+})
+
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/test.js b/packages/backend/src/routers/test.js
new file mode 100644
index 00000000..324def2b
--- /dev/null
+++ b/packages/backend/src/routers/test.js
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+
+// -----------------------------------------------------------------------//
+// GET /test
+// -----------------------------------------------------------------------//
+router.get('/test', async (req, res, next) => {
+ res.send('It\'s working!');
+})
+module.exports = router
diff --git a/packages/backend/src/routers/update-taskbar-items.js b/packages/backend/src/routers/update-taskbar-items.js
new file mode 100644
index 00000000..37cbb2a0
--- /dev/null
+++ b/packages/backend/src/routers/update-taskbar-items.js
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const config = require('../config.js');
+const { invalidate_cached_user } = require('../helpers');
+const router = new express.Router();
+const auth = require('../middleware/auth.js');
+const { DB_WRITE } = require('../services/database/consts.js');
+
+// -----------------------------------------------------------------------//
+// POST /update-taskbar-items
+// -----------------------------------------------------------------------//
+router.post('/update-taskbar-items', auth, express.json(), async (req, res, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ // check if user is verified
+ if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed)
+ return res.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'});
+
+ // modules
+ const db = req.services.get('database').get(DB_WRITE, 'ui');
+
+ // Check if req.body.items is set
+ if(!req.body.items)
+ return res.status(400).send({code: 'invalid_request', message: 'items is required.'});
+ // Check if req.body.items is an array
+ else if(!Array.isArray(req.body.items))
+ return res.status(400).send({code: 'invalid_request', message: 'items must be an array.'});
+
+ // insert into DB
+ await db.write(
+ `UPDATE user SET taskbar_items = ? WHERE user.id = ?`,
+ [
+ req.body.items ?? null,
+ req.user.id,
+ ]
+ )
+
+ invalidate_cached_user(req.user);
+
+ // send results to client
+ return res.send({});
+})
+module.exports = router
\ No newline at end of file
diff --git a/packages/backend/src/routers/version.js b/packages/backend/src/routers/version.js
new file mode 100644
index 00000000..f49e33d3
--- /dev/null
+++ b/packages/backend/src/routers/version.js
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const eggspress = require("../api/eggspress");
+
+module.exports = eggspress(['/version'], {
+ allowedMethods: ['GET'],
+ subdomain: 'api',
+ json: true,
+}, async (req, res, next) => {
+ const svc_puterVersion = req.services.get('puter-version');
+
+ const response = svc_puterVersion.get_version();
+
+ // Add user-friendly version information
+ {
+ response.version_text = response.version;
+ const components = response.version.split('-');
+ if ( components.length > 1 ) {
+ response.release_type = components[1];
+ if ( components[1] === 'rc' ) {
+ response.version_text =
+ `${components[0]} (Release Candidate ${components[2]})`;
+ }
+ else if ( components[1] === 'dev' ) {
+ response.version_text =
+ `${components[0]} (Development Build)`;
+ }
+ else if ( components[1] === 'beta' ) {
+ response.version_text =
+ `${components[0]} (Beta Release)`;
+ }
+ else if ( ! isNaN(components[1]) ) {
+ response.version_text = `${components[0]} (Build ${components[1]})`;
+ response.sub_version = components[1];
+ response.hash = components[2];
+ response.release_type = 'build';
+ }
+ if ( isNaN(components[1]) && components.length > 2 ) {
+ response.sub_version = components[2];
+ }
+ }
+ }
+
+ res.send(response);
+});
diff --git a/packages/backend/src/routers/whoami.js b/packages/backend/src/routers/whoami.js
new file mode 100644
index 00000000..e0266123
--- /dev/null
+++ b/packages/backend/src/routers/whoami.js
@@ -0,0 +1,161 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const { get_taskbar_items, is_shared_with_anyone, suggest_app_for_fsentry, get_app, get_descendants, id2uuid } = require('../helpers');
+const auth = require('../middleware/auth.js');
+const fs = require('../middleware/fs.js');
+const _path = require('path');
+const eggspress = require('../api/eggspress');
+const { Context } = require('../util/context');
+const { UserActorType } = require('../services/auth/Actor');
+
+// -----------------------------------------------------------------------//
+// GET /whoami
+// -----------------------------------------------------------------------//
+const WHOAMI_GET = eggspress('/whoami', {
+ subdomain: 'api',
+ auth2: true,
+ allowedMethods: ['GET'],
+}, async (req, res, next) => {
+ const actor = Context.get('actor');
+ if ( ! actor ) {
+ throw Error('actor not found in context');
+ }
+
+ const is_user = actor.type instanceof UserActorType;
+
+ // send user object
+ const details = {
+ username: req.user.username,
+ uuid: req.user.uuid,
+ email: req.user.email,
+ email_confirmed: req.user.email_confirmed,
+ requires_email_confirmation: req.user.requires_email_confirmation,
+ desktop_bg_url: req.user.desktop_bg_url,
+ desktop_bg_color: req.user.desktop_bg_color,
+ desktop_bg_fit: req.user.desktop_bg_fit,
+ is_temp: (req.user.password === null && req.user.email === null),
+ taskbar_items: await get_taskbar_items(req.user),
+ referral_code: req.user.referral_code,
+ };
+
+ if ( ! is_user ) {
+ // When apps call /whoami they should not see these attributes
+ // delete details.username;
+ // delete details.uuid;
+ delete details.email;
+ delete details.desktop_bg_url;
+ delete details.desktop_bg_color;
+ delete details.desktop_bg_fit;
+ delete details.taskbar_items;
+ }
+
+ res.send(details);
+})
+
+// -----------------------------------------------------------------------//
+// POST /whoami
+// -----------------------------------------------------------------------//
+const WHOAMI_POST = new express.Router();
+WHOAMI_POST.post('/whoami', auth, fs, express.json(), async (req, response, next)=>{
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ let desktop_items = [];
+
+ // check if user asked for desktop items
+ if(req.query.return_desktop_items === 1 || req.query.return_desktop_items === '1' || req.query.return_desktop_items === 'true'){
+ // by cached desktop id
+ if(req.user.desktop_id){
+ desktop_items = await db.read(
+ `SELECT * FROM fsentries
+ WHERE user_id = ? AND parent_uid = ?`,
+ [req.user.id, await id2uuid(req.user.desktop_id)]
+ )
+ }
+ // by desktop path
+ else{
+ desktop_items = await get_descendants(req.user.username +'/Desktop', req.user, 1, true);
+ }
+
+ // clean up desktop items and add some extra information
+ if(desktop_items.length > 0){
+ if(desktop_items.length > 0){
+ for (let i = 0; i < desktop_items.length; i++) {
+ if(desktop_items[i].id !== null){
+ // suggested_apps for files
+ if(!desktop_items[i].is_dir){
+ desktop_items[i].suggested_apps = await suggest_app_for_fsentry(desktop_items[i], {user: req.user});
+ }
+ // is_shared
+ desktop_items[i].is_shared = await is_shared_with_anyone(desktop_items[i].id);
+
+ // associated_app
+ if(desktop_items[i].associated_app_id){
+ const app = await get_app({id: desktop_items[i].associated_app_id})
+
+ // remove some privileged information
+ delete app.id;
+ delete app.approved_for_listing;
+ delete app.approved_for_opening_items;
+ delete app.godmode;
+ delete app.owner_user_id;
+ // add to array
+ desktop_items[i].associated_app = app;
+
+ }else{
+ desktop_items[i].associated_app = {};
+ }
+
+ // remove associated_app_id since it's sensitive info
+ // delete desktop_items[i].associated_app_id;
+ }
+ // id is sesitive info
+ delete desktop_items[i].id;
+ delete desktop_items[i].user_id;
+ delete desktop_items[i].bucket;
+ desktop_items[i].path = _path.join('/', req.user.username, desktop_items[i].name)
+ }
+ }
+ }
+ }
+
+ // send user object
+ response.send({
+ username: req.user.username,
+ uuid: req.user.uuid,
+ email: req.user.email,
+ email_confirmed: req.user.email_confirmed,
+ requires_email_confirmation: req.user.requires_email_confirmation,
+ desktop_bg_url: req.user.desktop_bg_url,
+ desktop_bg_color: req.user.desktop_bg_color,
+ desktop_bg_fit: req.user.desktop_bg_fit,
+ is_temp: (req.user.password === null && req.user.email === null),
+ taskbar_items: await get_taskbar_items(req.user),
+ desktop_items: desktop_items,
+ referral_code: req.user.referral_code,
+ });
+});
+
+module.exports = app => {
+ app.use(WHOAMI_GET);
+ app.use(WHOAMI_POST);
+};
\ No newline at end of file
diff --git a/packages/backend/src/routers/writeFile.js b/packages/backend/src/routers/writeFile.js
new file mode 100644
index 00000000..39c19b41
--- /dev/null
+++ b/packages/backend/src/routers/writeFile.js
@@ -0,0 +1,585 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const express = require('express');
+const router = new express.Router();
+const {uuid2fsentry, validate_signature_auth, get_url_from_req, sign_file} = require('../helpers');
+const multer = require('multer');
+const fs = require('../middleware/fs.js');
+const { NodePathSelector, NodeUIDSelector } = require('../filesystem/node/selectors');
+const eggspress = require('../api/eggspress');
+const { HLWrite } = require('../filesystem/hl_operations/hl_write');
+const { TYPE_DIRECTORY } = require('../filesystem/FSNodeContext');
+const { Context } = require('../util/context');
+const { Actor } = require('../services/auth/Actor');
+const { DB_WRITE } = require('../services/database/consts');
+const FSNodeParam = require('../api/filesystem/FSNodeParam');
+const { HLMove } = require('../filesystem/hl_operations/hl_move');
+const { HLCopy } = require('../filesystem/hl_operations/hl_copy');
+const { HLMkdir } = require('../filesystem/hl_operations/hl_mkdir');
+const { HLRemove } = require('../filesystem/hl_operations/hl_remove');
+
+// TODO: eggspressify
+
+// -----------------------------------------------------------------------//
+// POST /writeFile
+// -----------------------------------------------------------------------//
+module.exports = eggspress('/writeFile', {
+ fs: true,
+ files: ['file'],
+ allowedMethods: ['POST'],
+}, async (req, res, next) => {
+ // check subdomain
+ if(require('../helpers').subdomain(req) !== 'api')
+ next();
+
+ const log = req.services.get('log-service').create('writeFile');
+ const errors = req.services.get('error-service').create(log);
+
+ // validate URL signature
+ try{
+ validate_signature_auth(get_url_from_req(req), 'write');
+ }
+ catch(e){
+ return res.status(403).send(e);
+ }
+
+ log.info('writeFile context: ' + (
+ Context.get(undefined, { allow_fallback: true })
+ ).describe())
+ log.info('writeFile req context: ' + res.locals.ctx?.describe?.());
+
+ // Get fsentry
+ // todo this is done again in the following section, super inefficient
+ let requested_item = await uuid2fsentry(req.query.uid);
+
+ if ( ! requested_item ) {
+ return res.status(404).send({ error: 'Item not found' });
+ }
+
+ // check if requested_item owner is suspended
+ const owner_user = await require('../helpers').get_user({id: requested_item.user_id});
+
+ if ( ! owner_user ) {
+ errors.report('writeFile_no_owner', {
+ message: `User not found: ${requested_item.user_id}`,
+ trace: true,
+ alarm: true,
+ extra: {
+ requested_item,
+ body: req.body,
+ query: req.query,
+ }
+ })
+
+ return res.status(500).send({ error: 'User not found' });
+ }
+
+ if(owner_user.suspended)
+ return res.status(401).send({error: 'Account suspended'});
+
+ const db = req.services.get('database').get(DB_WRITE, 'filesystem');
+
+ // -----------------------------------------------------------------------//
+ // move
+ // -----------------------------------------------------------------------//
+ if(req.query.operation && req.query.operation === 'move'){
+ console.log(req.body)
+ const { get_user } = require('../helpers')
+ const _path = require('path');
+ const mime = require('mime-types');
+
+ // check if destination_write_url provided
+ if(!req.body.destination_write_url){
+ return res.status(400).send({
+ error:{
+ message: 'No destination specified.'
+ }
+ })
+ }
+
+ // check if destination_write_url is valid
+ try{
+ validate_signature_auth(req.body.destination_write_url, 'write');
+ }catch(e){
+ return res.status(403).send(e);
+ }
+
+ try{
+ const hl_move = new HLMove();
+
+ // TODO: [fs:operation:param-coercion]
+ const source_node = await (new FSNodeParam('uid')).consolidate({
+ req, getParam: () => req.query.uid
+ });
+
+ // TODO: [fs:operation:param-coercion]
+ const dest_node = await (new FSNodeParam('dest_path')).consolidate({
+ req, getParam: () => req.body.dest_path ?? req.body.destination_uid
+ });
+
+ const user = await get_user({id: await source_node.get('user_id')});
+
+ const opts = {
+ // TODO: [fs:normalize-writeFile-user]
+ user,
+ source: source_node,
+ destination_or_parent: dest_node,
+ overwrite: req.body.overwrite ?? false,
+ new_name: req.body.new_name,
+ new_metadata: req.body.new_metadata,
+ create_missing_parents: req.body.create_missing_parents,
+ };
+
+ // TODO: [fs:DRY-writeFile-context]
+ const r = await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => {
+ return await hl_move.run({
+ ...opts,
+ actor: Context.get('actor'),
+ });
+ });
+
+ return res.send({
+ ...r.moved,
+ old_path: r.old_path,
+ new_path: r.moved.path,
+ });
+ }catch(e){
+ console.log(e)
+ return res.status(400).send(e)
+ }
+ }
+
+ // -----------------------------------------------------------------------//
+ // copy
+ // -----------------------------------------------------------------------//
+ else if(req.query.operation && req.query.operation === 'copy'){
+ const {is_shared_with_anyone, suggest_app_for_fsentry, cp, validate_fsentry_name, convert_path_to_fsentry, uuid2fsentry, get_user, id2path, id2uuid} = require('../helpers')
+ const _path = require('path');
+ const mime = require('mime-types');
+
+ // check if destination_write_url provided
+ if(!req.body.destination_write_url){
+ return res.status(400).send({
+ error:{
+ message: 'No destination specified.'
+ }
+ })
+ }
+
+ // check if destination_write_url is valid
+ try{
+ validate_signature_auth(req.body.destination_write_url, 'write');
+ }catch(e){
+ return res.status(403).send(e);
+ }
+
+ const overwrite = req.body.overwrite ?? false;
+ const change_name = req.body.auto_rename ?? false;
+
+ // TODO: [fs:operation:param-coercion]
+ const source_node = await (new FSNodeParam('uid')).consolidate({
+ req, getParam: () => req.query.uid
+ });
+
+ // TODO: [fs:operation:param-coercion]
+ const dest_node = await (new FSNodeParam('dest_path')).consolidate({
+ req, getParam: () => req.body.dest_path ?? req.body.destination_uid
+ });
+
+ // Get user
+ let user = await get_user({id: await source_node.get('user_id')});
+
+ const opts = {
+ source: source_node,
+ destination_or_parent: dest_node,
+ dedupe_name: change_name,
+ overwrite,
+ user,
+ };
+
+ let new_fsentries
+ try{
+ const hl_copy = new HLCopy();
+
+ const r = await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => {
+ return await hl_copy.run({
+ ...opts,
+ actor: Context.get('actor'),
+ });
+ });
+ return res.send([r]);
+ }catch(e){
+ console.log(e)
+ return res.status(400).send(e)
+ }
+ }
+
+ // -----------------------------------------------------------------------//
+ // mkdir
+ // -----------------------------------------------------------------------//
+ else if(req.query.operation && req.query.operation === 'mkdir'){
+ const {mkdir, uuid2fsentry, get_user, id2path} = require('../helpers')
+
+ // name is required
+ if(!req.body.name){
+ return res.status(400).send({
+ error:{
+ message: 'Name is required.'
+ }
+ })
+ }
+
+ // TODO: [fs:operation:param-coercion]
+ const source_node = await (new FSNodeParam('uid')).consolidate({
+ req, getParam: () => req.query.uid
+ });
+
+
+ // Get user
+ let user = await get_user({id: await source_node.get('user_id')});
+
+ // Create new dir and return
+ try{
+ // TODO: [fs:remove-old-methods]
+ const hl_mkdir = new HLMkdir();
+ const r = await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => {
+ return await hl_mkdir.run({
+ parent: source_node,
+ path: req.body.name,
+ overwrite: false,
+ dedupe_name: req.body.dedupe_name ?? false,
+ user,
+ actor: Context.get('actor'),
+ });
+ });
+ const newdir_node = await req.fs.node(new NodeUIDSelector(r.uid));
+ return res.send(await sign_file(
+ await newdir_node.get('entry'), 'write'));
+ }catch(e){
+ console.log(e)
+ return res.status(400).send(e);
+ }
+ }
+
+ // -----------------------------------------------------------------------//
+ // Trash
+ // -----------------------------------------------------------------------//
+ if(req.query.operation && req.query.operation === 'trash'){
+ const {validate_fsentry_name, convert_path_to_fsentry, uuid2fsentry, get_user, id2path, id2uuid} = require('../helpers')
+ const _path = require('path');
+ const mime = require('mime-types');
+
+ // Get fsentry
+ const fs = req.services.get('filesystem');
+
+ // TODO: [fs:move-FSNodeParam]
+ const node = await (new FSNodeParam('path')).consolidate({
+ req, getParam: () => req.query.uid
+ });
+
+ // Get user
+ // TODO: [avoid-database-user-id]
+ let user = await get_user({id: await node.get('user_id')});
+
+ // metadata for trashed file
+ const new_name = await node.get('uid');
+ const metadata = {
+ original_name: await node.get('name'),
+ original_path: await node.get('path'),
+ trashed_ts: Math.round(Date.now() / 1000),
+ };
+
+ // Get Trash fsentry
+ const trash = await fs.node(
+ new NodePathSelector('/' + user.username + '/Trash')
+ );
+ // let trash_path = '/' + user.username + '/Trash';
+ // let trash = await convert_path_to_fsentry(trash_path);
+
+ console.log('what is trash?', trash);
+
+ const hl_move = new HLMove();
+ await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => {
+ await hl_move.run({
+ source: node,
+ destination_or_parent: trash,
+ // TODO: [fs:decouple-user]
+ user,
+ actor: Context.get('actor'),
+ new_name: new_name,
+ new_metadata: metadata,
+ });
+ });
+
+ // No Trash?
+ if(!trash){
+ return res.status(400).send({
+ error:{
+ message: 'No Trash directory found.'
+ }
+ })
+ }
+
+ return res.status(200).send({
+ message: 'Item trashed'
+ })
+ }
+ // -----------------------------------------------------------------------//
+ // Rename
+ // -----------------------------------------------------------------------//
+ if(req.query.operation && req.query.operation === 'rename'){
+ const {validate_fsentry_name, uuid2fsentry, get_app, id2path} = require('../helpers')
+ const _path = require('path');
+ const mime = require('mime-types');
+
+ // new_name validation
+ try{
+ validate_fsentry_name(req.body.new_name)
+ }catch(e){
+ return res.status(400).send({
+ error:{
+ message: e.message
+ }
+ });
+ }
+
+ // Get fsentry
+ let fsentry = await uuid2fsentry(req.query.uid);
+
+ // Not found
+ if(fsentry === false){
+ return res.status(400).send({
+ error:{
+ message: 'No entry found with this uid'
+ }
+ })
+ }
+
+ // Immutable?
+ if(fsentry.immutable){
+ return res.status(400).send({
+ error:{
+ message: 'Immutable: cannot rename.'
+ }
+ })
+ }
+
+ let res1;
+
+ // parent is root
+ if(fsentry.parent_uid === null){
+ try{
+ res1 = await db.read(
+ `SELECT uuid FROM fsentries WHERE parent_uid IS NULL AND name = ? AND id != ? LIMIT 1`,
+ [
+ //name
+ req.body.new_name,
+ fsentry.id,
+ ]);
+ }catch(e){
+ console.log(e)
+ }
+ }
+ // parent is regular dir
+ else{
+ res1 = await db.read(
+ `SELECT uuid FROM fsentries WHERE parent_uid = ? AND name = ? AND id != ? LIMIT 1`,
+ [
+ //parent_uid
+ fsentry.parent_uid,
+ //name
+ req.body.new_name,
+ fsentry.id,
+ ]);
+ }
+ if(res1[0]){
+ return res.status(400).send({
+ error:{
+ message: 'An entry with the same name exists under target path.'
+ }
+ })
+ }
+
+ // old path
+ const old_path = await id2path(fsentry.id);
+
+ // update `name`
+ await db.write(
+ `UPDATE fsentries SET name = ? WHERE id = ?`,
+ [req.body.new_name, fsentry.id]
+ )
+
+ // new path
+ const new_path = _path.join(_path.dirname(old_path), req.body.new_name);
+
+ // associated_app
+ let associated_app;
+ if(fsentry.associated_app_id){
+ const app = await get_app({id: fsentry.associated_app_id})
+ // remove some privileged information
+ delete app.id;
+ delete app.approved_for_listing;
+ delete app.approved_for_opening_items;
+ delete app.godmode;
+ delete app.owner_user_id;
+ // add to array
+ associated_app = app;
+ }else{
+ associated_app = {};
+ }
+
+ // send the fsentry of the new object created
+ const contentType = mime.contentType(req.body.new_name)
+ const return_obj = {
+ uid: fsentry.uuid,
+ name: req.body.new_name,
+ is_dir: fsentry.is_dir,
+ path: new_path,
+ old_path: old_path,
+ type: contentType ? contentType : null,
+ associated_app: associated_app,
+ original_client_socket_id: req.body.original_client_socket_id,
+ };
+
+ // send realtime success msg to client
+ let socketio = require('../socketio.js').getio();
+ if(socketio){
+ socketio.to(fsentry.user_id).emit('item.renamed', return_obj)
+ }
+
+ return res.send(return_obj);
+ }
+
+ // -----------------------------------------------------------------------//
+ // Delete
+ // -----------------------------------------------------------------------//
+ if(req.query.operation && req.query.operation === 'delete'){
+ const {get_user, uuid2fsentry, id2path} = require('../helpers')
+ const _path = require('path');
+ const mime = require('mime-types');
+
+ // TODO: [fs:operation:param-coercion]
+ const source_node = await (new FSNodeParam('uid')).consolidate({
+ req, getParam: () => req.query.uid
+ });
+
+ const user = await get_user({id: await source_node.get('user_id')});
+
+ // Delete
+ try{
+ const hl_remove = new HLRemove();
+ await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => {
+ await hl_remove.run({
+ target: source_node,
+ user,
+ actor: Context.get('actor'),
+ });
+ });
+ }catch(error){
+ console.log(error)
+ res.status(400).send(error);
+ }
+
+ // Send success msg
+ return res.send();
+ }
+
+ // -----------------------------------------------------------------------//
+ // Write
+ // -----------------------------------------------------------------------//
+ else{
+ // modules
+ const {uuid2fsentry, id2path} = require('../helpers')
+ const write = require('../filesystem/operations/write.js');
+ const _path = require('path');
+
+ // Check if files were uploaded
+ if(!req.files)
+ return res.status(400).send('No files uploaded');
+
+ // Get fsentry
+ let fsentry, dirname;
+ let node;
+
+ try{
+ node = await req.fs.node(new NodeUIDSelector(req.query.uid));
+ dirname = (await node.get('type') !== TYPE_DIRECTORY
+ ? _path.dirname.bind(_path) : a=>a)(await node.get('path'));
+ }catch(e){
+ console.log(e)
+ req.__error_source = e;
+ return res.status(500).send(e);
+ }
+
+ const user = await (async () => {
+ const { get_user } = require('../helpers');
+ const user_id = await node.get('user_id')
+ return await get_user({ id: user_id });
+ })();
+ Context.set('user', user);
+
+ const dirNode = await req.fs.node(new NodePathSelector(dirname));
+
+ const actor = Actor.adapt(user);
+
+ const context = Context.get().sub({
+ actor, user,
+ });
+
+ log.noticeme('writeFile: ' + context.describe());
+
+ // Upload files one by one
+ const returns = [];
+ for ( const uploaded_file of req.files ) {
+ try{
+ await context.arun(async () => {
+ const hl_write = new HLWrite();
+ const ret_obj = await hl_write.run({
+ destination_or_parent: dirNode,
+ specified_name: await node.get('type') === TYPE_DIRECTORY
+ ? req.body.name : await node.get('name'),
+ fallback_name: uploaded_file.originalname,
+ overwrite: true,
+ user: user,
+ actor,
+
+ file: uploaded_file,
+ });
+
+ // add signature to object
+ ret_obj.signature = await sign_file(ret_obj, 'write');
+
+ // send results back to app
+ returns.push(ret_obj);
+ });
+ }catch(error){
+ req.__error_source = error;
+ console.log(error)
+ return res.contentType('application/json').status(500).send(error);
+ }
+ }
+
+ if ( returns.length === 1 ) {
+ return res.send(returns[0]);
+ }
+
+ return res.send(returns);
+ }
+});
diff --git a/packages/backend/src/server b/packages/backend/src/server
new file mode 100644
index 00000000..e69de29b
diff --git a/packages/backend/src/services/AppInformationService.js b/packages/backend/src/services/AppInformationService.js
new file mode 100644
index 00000000..9d928b2b
--- /dev/null
+++ b/packages/backend/src/services/AppInformationService.js
@@ -0,0 +1,288 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { asyncSafeSetInterval } = require("../util/promise");
+const { MINUTE, SECOND } = require("../util/time");
+const { origin_from_url } = require("../util/urlutil");
+const { DB_READ } = require("./database/consts");
+
+const uuidv4 = require('uuid').v4;
+
+class AppInformationService {
+ constructor ({ services }) {
+ this.services = services;
+ this.log = services.get('log-service').create('app-info');
+
+ this.collections = {};
+ this.collections.recent = [];
+
+ this.tags = {};
+
+ (async () => {
+ // await new Promise(rslv => setTimeout(rslv, 500))
+
+ await this._refresh_app_cache();
+ asyncSafeSetInterval(async () => {
+ this._refresh_app_cache();
+ }, 30 * 1000);
+
+ await this._refresh_app_stats();
+ asyncSafeSetInterval(async () => {
+ this._refresh_app_stats();
+ }, 120 * 1000);
+
+ // This stat is more expensive so we don't update it as often
+ await this._refresh_app_stat_referrals();
+ asyncSafeSetInterval(async () => {
+ this._refresh_app_stat_referrals();
+ }, 15 * MINUTE);
+
+ await this._refresh_recent_cache();
+ asyncSafeSetInterval(async () => {
+ this._refresh_recent_cache();
+ }, 120 * 1000);
+
+ await this._refresh_tags();
+ asyncSafeSetInterval(async () => {
+ this._refresh_tags();
+ } , 120 * 1000);
+ })();
+ }
+
+ async get_stats (app_uid) {
+ const db = this.services.get('database').get(DB_READ, 'apps');
+
+ const key_open_count = `apps:open_count:uid:${app_uid}`;
+ let open_count = kv.get(key_open_count);
+ if ( ! open_count ) {
+ open_count = (await db.read(
+ `SELECT COUNT(_id) AS open_count FROM app_opens WHERE app_uid = ?`,
+ [app_uid]
+ ))[0].open_count;
+ }
+
+ // TODO: cache
+ const key_user_count = `apps:user_count:uid:${app_uid}`;
+ let user_count = kv.get(key_user_count);
+ if ( ! user_count ) {
+ user_count = (await db.read(
+ `SELECT COUNT(DISTINCT user_id) AS user_count FROM app_opens WHERE app_uid = ?`,
+ [app_uid]
+ ))[0].user_count;
+ }
+
+ const key_referral_count = `apps:referral_count:uid:${app_uid}`;
+ let referral_count = kv.get(key_referral_count);
+ if ( ! referral_count ) {
+ // NOOP: this operation is expensive so if it's not cached
+ // we simply won't report it
+ }
+
+ return {
+ open_count,
+ user_count,
+ referral_count,
+ };
+ }
+
+ async _refresh_app_cache () {
+ this.log.tick('refresh app cache');
+
+ const db = this.services.get('database').get(DB_READ, 'apps');
+
+ let apps = await db.read('SELECT * FROM apps');
+ for (let index = 0; index < apps.length; index++) {
+ const app = apps[index];
+ kv.set('apps:name:' + app.name, app);
+ kv.set('apps:id:' + app.id, app);
+ kv.set('apps:uid:' + app.uid, app);
+ }
+ }
+
+ async _refresh_app_stats () {
+ this.log.tick('refresh app stats');
+
+ const db = this.services.get('database').get(DB_READ, 'apps');
+
+ // you know, it's interesting that I need to specify 'uid'
+ // meanwhile static analysis of the code could determine that
+ // no other column here is ever used.
+ // I'm not suggesting a specific solution for here, but it's
+ // interesting to think about.
+
+ const apps = await db.read(`SELECT uid FROM apps`);
+
+ for ( const app of apps ) {
+ const key_open_count = `apps:open_count:uid:${app.uid}`;
+ const { open_count } = (await db.read(
+ `SELECT COUNT(_id) AS open_count FROM app_opens WHERE app_uid = ?`,
+ [app.uid]
+ ))[0];
+ kv.set(key_open_count, open_count);
+
+ const key_user_count = `apps:user_count:uid:${app.uid}`;
+ const { user_count } = (await db.read(
+ `SELECT COUNT(DISTINCT user_id) AS user_count FROM app_opens WHERE app_uid = ?`,
+ [app.uid]
+ ))[0];
+ kv.set(key_user_count, user_count);
+ }
+ }
+
+ async _refresh_app_stat_referrals () {
+ this.log.tick('refresh app stat referrals');
+
+ const db = this.services.get('database').get(DB_READ, 'apps');
+
+ const apps = await db.read(`SELECT uid, index_url FROM apps`);
+
+ for ( const app of apps ) {
+ const sql =
+ `SELECT COUNT(id) AS referral_count FROM user WHERE referrer = ?`;
+
+ const origin = origin_from_url(app.index_url);
+
+ // only count the referral if the origin hashes to the app's uid
+ const svc_auth = this.services.get('auth');
+ const expected_uid = await svc_auth.app_uid_from_origin(origin);
+ if ( expected_uid !== app.uid ) {
+ continue;
+ }
+
+ const key_referral_count = `apps:referral_count:uid:${app.uid}`;
+ const { referral_count } = (await db.read(
+ `SELECT COUNT(id) AS referral_count FROM user WHERE referrer LIKE ?`,
+ [origin + '%']
+ ))[0];
+
+ if ( app.uid === 'app-eeec9a28-0eb1-5b63-a2dd-b99a8a3cf4c3' ) {
+ console.log('app?', app);
+ console.log('REFERRAL COUNT', referral_count, {
+ sql,
+ index_url: app.index_url,
+ });
+ }
+
+ kv.set(key_referral_count, referral_count);
+ }
+
+ this.log.info('DONE refresh app stat referrals');
+ }
+
+ async _refresh_recent_cache () {
+ const app_keys = kv.keys(`apps:uid:*`);
+ // console.log('APP KEYS', app_keys);
+
+ let apps = [];
+ for ( const key of app_keys ) {
+ const app = kv.get(key);
+ apps.push(app);
+ }
+
+ apps = apps.filter(app => app.approved_for_listing);
+ apps.sort((a, b) => {
+ return b.timestamp - a.timestamp;
+ });
+
+ this.collections.recent = apps.map(app => app.uid).slice(0, 50);
+ }
+
+ async _refresh_tags () {
+ const app_keys = kv.keys(`apps:uid:*`);
+ // console.log('APP KEYS', app_keys);
+
+ let apps = [];
+ for ( const key of app_keys ) {
+ const app = kv.get(key);
+ apps.push(app);
+ }
+
+ apps = apps.filter(app => app.approved_for_listing);
+ apps.sort((a, b) => {
+ return b.timestamp - a.timestamp;
+ });
+
+ const new_tags = {};
+
+ for ( const app of apps ) {
+ const app_tags = (app.tags ?? '').split(',')
+ .map(tag => tag.trim())
+ .filter(tag => tag.length > 0);
+
+ for ( const tag of app_tags ) {
+ if ( ! new_tags[tag] ) new_tags[tag] = {};
+ new_tags[tag][app.uid] = true;
+ }
+ }
+
+ for ( const tag in new_tags ) {
+ new_tags[tag] = Object.keys(new_tags[tag]);
+ }
+
+ this.tags = new_tags;
+ }
+
+ async delete_app (app_uid, app) {
+ const db = this.services.get('database').get(DB_READ, 'apps');
+
+ app = app ?? kv.get('apps:uid:' + app_uid);
+ if ( ! app ) {
+ app = (await db.read(
+ `SELECT * FROM apps WHERE uid = ?`,
+ [app_uid]
+ ))[0];
+ }
+
+ if ( ! app ) {
+ throw new Error('app not found');
+ }
+
+ await db.write(
+ `DELETE FROM apps WHERE uid = ? LIMIT 1`,
+ [app_uid]
+ );
+
+ // remove from caches
+ kv.del('apps:name:' + app.name);
+ kv.del('apps:id:' + app.id);
+ kv.del('apps:uid:' + app.uid);
+
+ // remove from recent
+ const index = this.collections.recent.indexOf(app_uid);
+ if ( index >= 0 ) {
+ this.collections.recent.splice(index, 1);
+ }
+
+ // remove from tags
+ const app_tags = (app.tags ?? '').split(',')
+ .map(tag => tag.trim())
+ .filter(tag => tag.length > 0);
+ for ( const tag of app_tags ) {
+ if ( ! this.tags[tag] ) continue;
+ const index = this.tags[tag].indexOf(app_uid);
+ if ( index >= 0 ) {
+ this.tags[tag].splice(index, 1);
+ }
+ }
+
+ }
+}
+
+module.exports = {
+ AppInformationService,
+};
diff --git a/packages/backend/src/services/BaseService.js b/packages/backend/src/services/BaseService.js
new file mode 100644
index 00000000..b2bce602
--- /dev/null
+++ b/packages/backend/src/services/BaseService.js
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+const NOOP = async () => {};
+
+class BaseService extends AdvancedBase {
+ constructor (service_resources, ...a) {
+ const { services, config, my_config, name, args } = service_resources;
+ super(service_resources, ...a);
+
+ this.args = args;
+ this.service_name = name || this.constructor.name;
+ this.services = services;
+ this.config = my_config;
+ this.global_config = config;
+
+ if ( this.global_config.server_id === '' ) {
+ this.global_config.server_id = 'local';
+ }
+ }
+
+ async construct () {
+ await (this._construct || NOOP).call(this, this.args);
+ }
+
+ async init () {
+ const services = this.services;
+ this.log = services.get('log-service').create(this.service_name);
+ this.errors = services.get('error-service').create(this.log);
+
+ await (this._init || NOOP).call(this, this.args);
+ }
+
+ async __on (id, args) {
+ const handler = this.__get_event_handler(id);
+
+ return await handler(id, args);
+ }
+
+ __get_event_handler (id) {
+ return this[`__on_${id}`]?.bind?.(this)
+ || this.constructor[`__on_${id}`]?.bind?.(this.constructor)
+ || NOOP;
+ }
+}
+
+module.exports = BaseService;
diff --git a/packages/backend/src/services/ClientOperationService.js b/packages/backend/src/services/ClientOperationService.js
new file mode 100644
index 00000000..58c93deb
--- /dev/null
+++ b/packages/backend/src/services/ClientOperationService.js
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Context } = require("../util/context");
+
+const CONTEXT_KEY = Context.make_context_key('operation-trace');
+class ClientOperationTracker {
+ constructor (parameters) {
+ this.name = parameters.name || 'untitled';
+ this.tags = parameters.tags || [];
+ this.frame = parameters.frame || null;
+ this.metadata = parameters.metadata || {};
+ this.objects = parameters.objects || [];
+ }
+}
+
+class ClientOperationService {
+ constructor ({ services }) {
+ this.operations_ = [];
+ }
+
+ async add_operation (parameters) {
+ const tracker = new ClientOperationTracker(parameters);
+
+ return tracker;
+ }
+
+ ckey (key) {
+ return CONTEXT_KEY + ':' + key;
+ }
+}
+
+module.exports = {
+ ClientOperationService,
+};
diff --git a/packages/backend/src/services/CommandService.js b/packages/backend/src/services/CommandService.js
new file mode 100644
index 00000000..b7891979
--- /dev/null
+++ b/packages/backend/src/services/CommandService.js
@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("./BaseService");
+
+class Command {
+ constructor(spec) {
+ this.spec_ = spec;
+ }
+
+ async execute(args, log) {
+ log = log ?? console;
+ const { id, name, description, handler } = this.spec_;
+ try {
+ await handler(args, log);
+ } catch (err) {
+ log.error(`command ${name ?? id} failed: ${err.message}`);
+ log.error(err.stack);
+ }
+ }
+}
+
+class CommandService extends BaseService {
+ async _construct () {
+ this.commands_ = [];
+ }
+ async _init () {
+ this.commands_.push(new Command({
+ id: 'help',
+ description: 'show this help',
+ handler: (args, log) => {
+ log.log(`available commands:`);
+ for (const command of this.commands_) {
+ log.log(`- ${command.spec_.id}: ${command.spec_.description}`);
+ }
+ }
+ }));
+ }
+
+ registerCommands(serviceName, commands) {
+ for (const command of commands) {
+ this.log.info(`registering command ${serviceName}:${command.id}`);
+ this.commands_.push(new Command({
+ ...command,
+ id: `${serviceName}:${command.id}`,
+ }));
+ }
+ }
+
+ async executeCommand(args, log) {
+ const [commandName, ...commandArgs] = args;
+ const command = this.commands_.find(c => c.spec_.id === commandName);
+ if ( ! command ) {
+ log.error(`unknown command: ${commandName}`);
+ return;
+ }
+ await globalThis.root_context.arun(async () => {
+ await command.execute(commandArgs, log);
+ });
+ }
+
+ async executeRawCommand(text, log) {
+ // TODO: add obvious-json as a tokenizer
+ const args = text.split(/\s+/);
+ await this.executeCommand(args, log);
+ }
+}
+
+module.exports = {
+ CommandService
+};
\ No newline at end of file
diff --git a/packages/backend/src/services/ConfigurableCountingService.js b/packages/backend/src/services/ConfigurableCountingService.js
new file mode 100644
index 00000000..18907bb1
--- /dev/null
+++ b/packages/backend/src/services/ConfigurableCountingService.js
@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+var crypto = require('crypto')
+const BaseService = require("./BaseService");
+const { Context } = require("../util/context");
+const { DB_WRITE } = require('./database/consts');
+
+const hash = v => {
+ var sum = crypto.createHash('sha1');
+ sum.update('foo');
+ return sum.digest();
+}
+
+class ConfigurableCountingService extends BaseService {
+ static counting_types = {
+ gpt: {
+ category: [
+ {
+ name: 'model',
+ type: 'string',
+ }
+ ],
+ values: [
+ {
+ name: 'input_tokens',
+ type: 'uint',
+ },
+ {
+ name: 'output_tokens',
+ type: 'uint',
+ }
+ ]
+ },
+ dalle: {
+ category: [
+ {
+ name: 'model',
+ type: 'string',
+ },
+ {
+ name: 'quality',
+ type: 'string',
+ },
+ {
+ name: 'resolution',
+ type: 'string',
+ }
+ ],
+ }
+ };
+
+ static sql_columns = {
+ uint: [
+ 'value_uint_1',
+ 'value_uint_2',
+ 'value_uint_3',
+ ],
+ }
+
+ async _init () {
+ this.db = this.services.get('database').get(DB_WRITE, 'counting');
+ }
+
+ async increment ({ service_name, service_type, values }) {
+ values = values ? {...values} : {};
+
+ const now = new Date();
+ const year = now.getUTCFullYear();
+ const month = now.getUTCMonth() + 1;
+
+ const counting_type = this.constructor.counting_types[service_type];
+ if ( ! counting_type ) {
+ throw new Error(`unknown counting type ${service_type}`);
+ }
+
+ const available_columns = {};
+ for ( const k in this.constructor.sql_columns ) {
+ available_columns[k] = [...this.constructor.sql_columns[k]];
+ }
+
+ const custom_col_names = counting_type.values.map((value, index) => {
+ const column = available_columns[value.type].shift();
+ if ( ! column ) {
+ // TODO: this could be an init check on all the available service types
+ throw new Error(`no more available columns for type ${value.type}`);
+ }
+ return column;
+ });
+
+ const custom_col_values = counting_type.values.map((value, index) => {
+ return values[value.name];
+ });
+
+ // `pricing_category` is a JSON field. Keys from `values` used for
+ // the pricing category will be removed from ths `values` object
+ const pricing_category = {};
+ for ( const category of counting_type.category ) {
+ pricing_category[category.name] = values[category.name];
+ delete values[category.name];
+ }
+
+ // `JSON.stringify` cannot be used here because it does not sort
+ // the keys.
+ const pricing_category_str = counting_type.category.map((category) => {
+ return `${category.name}:${pricing_category[category.name]}`;
+ }).join(',');
+
+ const pricing_category_hash = hash(pricing_category_str);
+
+ const actor = Context.get('actor');
+ const actor_key = actor.uid;
+
+ const required_data = {
+ year, month, service_name, service_type,
+ actor_key, pricing_category_hash,
+ pricing_category: JSON.stringify(pricing_category),
+ };
+
+ const sql =
+ `INSERT INTO monthly_usage_counts (${
+ Object.keys(required_data).join(', ')
+ }, count, ${
+ custom_col_names.join(', ')
+ }) ` +
+ `VALUES (${
+ Object.keys(required_data).map(() => '?').join(', ')
+ }, 1, ${custom_col_values.map(() => '?').join(', ')}) ` +
+ `ON DUPLICATE KEY UPDATE count = count + 1${
+ custom_col_names.length > 0 ? ', ' : ''
+ } ${
+ custom_col_names.map((name) => `${name} = ${name} + ?`).join(', ')
+ }`;
+
+ const value_array = [
+ ...Object.values(required_data),
+ ...custom_col_values,
+ ...custom_col_values,
+ ]
+
+ console.log('SQL QUERY', sql, value_array);
+
+ await this.db.write(sql, value_array);
+ }
+}
+
+module.exports = {
+ ConfigurableCountingService,
+};
diff --git a/packages/backend/src/services/Container.js b/packages/backend/src/services/Container.js
new file mode 100644
index 00000000..43a40d85
--- /dev/null
+++ b/packages/backend/src/services/Container.js
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const config = require("../config");
+const { CompositeError } = require("../util/errorutil");
+const { TeePromise } = require("../util/promise");
+
+// 17 lines of code instead of an entire dependency-injection framework
+class Container {
+ constructor () {
+ this.instances_ = {};
+ this.ready = new TeePromise();
+ }
+ registerService (name, cls, args) {
+ const my_config = config.services?.[name] || {};
+ this.instances_[name] = cls.getInstance
+ ? cls.getInstance({ services: this, config, my_config, name, args })
+ : new cls({ services: this, config, my_config, name, args }) ;
+ }
+ set (name, instance) { this.instances_[name] = instance; }
+ get (name, opts) {
+ if ( this.instances_[name] ) {
+ return this.instances_[name];
+ }
+ if ( ! opts?.optional ) {
+ throw new Error(`missing service: ${name}`);
+ }
+ }
+ has (name) { return !! this.instances_[name]; }
+ get values () {
+ const values = {};
+ for ( const k in this.instances_ ) {
+ let k2 = k;
+
+ // Replace lowerCamelCase with underscores
+ // (just an idea; more effort than it's worth right now)
+ // let k2 = k.replace(/([a-z])([A-Z])/g, '$1_$2')
+
+ // Replace dashes with underscores
+ k2 = k2.replace(/-/g, '_');
+ // Convert to lower case
+ k2 = k2.toLowerCase();
+
+ values[k2] = this.instances_[k];
+ }
+ return this.instances_;
+ }
+
+ async init () {
+ for ( const k in this.instances_ ) {
+ console.log(`constructing ${k}`);
+ await this.instances_[k].construct();
+ }
+ const init_failures = [];
+ for ( const k in this.instances_ ) {
+ console.log(`initializing ${k}`);
+ try {
+ await this.instances_[k].init();
+ } catch (e) {
+ init_failures.push({ k, e });
+ }
+ }
+
+ if ( init_failures.length ) {
+ console.error('init failures', init_failures);
+ throw new CompositeError(
+ `failed to initialize these services: ` +
+ init_failures.map(({ k }) => k).join(', '),
+ init_failures.map(({ k, e }) => e)
+ );
+ }
+ }
+
+ async emit (id, ...args) {
+ if ( this.logger ) {
+ this.logger.noticeme(`services:event ${id}`, { args });
+ }
+ const promises = [];
+ for ( const k in this.instances_ ) {
+ if ( this.instances_[k].__on ) {
+ promises.push(this.instances_[k].__on(id, ...args));
+ }
+ }
+ await Promise.all(promises);
+ }
+}
+
+class ProxyContainer {
+ constructor (delegate) {
+ this.delegate = delegate;
+ this.instances_ = {};
+ }
+ set (name, instance) {
+ this.instances_[name] = instance;
+ }
+ get (name) {
+ if ( this.instances_.hasOwnProperty(name) ) {
+ return this.instances_[name];
+ }
+ return this.delegate.get(name);
+ }
+ has (name) {
+ if ( this.instances_.hasOwnProperty(name) ) {
+ return true;
+ }
+ return this.delegate.has(name);
+ }
+ get values () {
+ const values = {};
+ Object.assign(values, this.delegate.values);
+ for ( const k in this.instances_ ) {
+ let k2 = k;
+
+ // Replace lowerCamelCase with underscores
+ // (just an idea; more effort than it's worth right now)
+ // let k2 = k.replace(/([a-z])([A-Z])/g, '$1_$2')
+
+ // Replace dashes with underscores
+ k2 = k2.replace(/-/g, '_');
+ // Convert to lower case
+ k2 = k2.toLowerCase();
+
+ values[k2] = this.instances_[k];
+ }
+ return values;
+ }
+}
+
+module.exports = { Container, ProxyContainer };
diff --git a/packages/backend/src/services/ContextInitService.js b/packages/backend/src/services/ContextInitService.js
new file mode 100644
index 00000000..ac30cfb3
--- /dev/null
+++ b/packages/backend/src/services/ContextInitService.js
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Context } = require("../util/context");
+const BaseService = require("./BaseService");
+
+// DRY: (2/3) - src/util/context.js; move install() to base class
+class ContextInitExpressMiddleware {
+ constructor () {
+ this.value_initializers_ = [];
+ }
+ register_initializer (initializer) {
+ this.value_initializers_.push(initializer);
+ }
+ install (app) {
+ app.use(this.run.bind(this));
+ }
+ async run (req, res, next) {
+ const x = Context.get();
+ for ( const initializer of this.value_initializers_ ) {
+ if ( initializer.value ) {
+ x.set(initializer.key, initializer.value);
+ } else if ( initializer.async_factory ) {
+ x.set(initializer.key, await initializer.async_factory());
+ }
+ }
+ next();
+ }
+}
+
+class ContextInitService extends BaseService {
+ _construct () {
+ this.mw = new ContextInitExpressMiddleware();
+ }
+ register_value (key, value) {
+ this.mw.register_initializer({
+ key, value,
+ });
+ }
+ register_async_factory (key, factory) {
+ this.mw.register_initializer({
+ key, async_factory,
+ });
+ }
+ async ['__on_install.middlewares.context-aware'] (_, { app }) {
+ this.mw.install(app);
+ await this.services.emit('install.context-initializers');
+ }
+}
+
+module.exports = {
+ ContextInitService
+};
\ No newline at end of file
diff --git a/packages/backend/src/services/DevConsoleService.js b/packages/backend/src/services/DevConsoleService.js
new file mode 100644
index 00000000..b8864073
--- /dev/null
+++ b/packages/backend/src/services/DevConsoleService.js
@@ -0,0 +1,200 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { consoleLogManager } = require('../util/consolelog');
+const BaseService = require('./BaseService');
+
+class DevConsoleService extends BaseService {
+ _construct () {
+ this.static_lines = [];
+ this.widgets = [];
+ this.identifiers = {};
+ }
+
+ turn_on_the_warning_lights () {
+ this.add_widget(() => {
+ return `\x1B[31;1m\x1B[5m *** ${
+ Array(3).fill('WARNING').join(' ** ')
+ } ***\x1B[0m`;
+ });
+ }
+
+ add_widget (outputter, opt_id) {
+ this.widgets.push(outputter);
+ if ( opt_id ) {
+ this.identifiers[opt_id] = outputter;
+ }
+ }
+
+ remove_widget (id_or_outputter) {
+ if ( typeof id_or_outputter === 'string' ) {
+ id_or_outputter = this.identifiers[id_or_outputter];
+ }
+ this.widgets = this.widgets.filter(w => w !== id_or_outputter);
+ }
+
+ update_ () {
+ this.static_lines = [];
+ // if a widget throws an error we MUST remove it;
+ // it's probably a stack overflow because it's printing.
+ const to_remove = [];
+ for ( const w of this.widgets ) {
+ let output; try {
+ output = w();
+ } catch ( e ) {
+ consoleLogManager.log_raw('error', e);
+ to_remove.push(w);
+ continue;
+ }
+ output = Array.isArray(output) ? output : [output];
+ this.static_lines.push(...output);
+ }
+ for ( const w of to_remove ) {
+ this.remove_widget(w);
+ }
+ }
+
+ async _init () {
+ const services = this.services;
+ // await services.ready;
+ const commands = services.get('commands');
+
+ const readline = require('readline');
+ const rl = readline.createInterface({
+ input: process.stdin,
+ output: process.stdout,
+ prompt: 'puter> ',
+ terminal: true,
+ });
+ rl.on('line', async (input) => {
+ this._before_cmd();
+ if ( input.startsWith('ev') ) {
+ eval(input.slice(3));
+ } else {
+ await commands.executeRawCommand(input, console);
+ }
+ this._after_cmd();
+ // rl.prompt();
+ });
+
+ this._before_cmd = () => {
+ rl.pause();
+ rl.output.write('\x1b[1A\r');
+ rl.output.write('\x1b[2K\r');
+ console.log(
+ `\x1B[33m` +
+ this.generateSeparator(`[ Command Output ]`) +
+ `\x1B[0m`
+ );
+ }
+
+ this._after_cmd = () => {
+ console.log(
+ `\x1B[33m` +
+ this.generateEnd() +
+ `\x1B[0m`
+ );
+ }
+
+ this._pre_write = () => {
+ rl.pause();
+ process.stdout.write('\x1b[0m');
+ rl.output.write('\x1b[2K\r');
+ for (let i = 0; i < this.static_lines.length + 1; i++) {
+ process.stdout.write('\x1b[1A'); // Move cursor up one line
+ process.stdout.write('\x1b[2K'); // Clear the line
+ }
+ }
+
+ this._post_write = () => {
+ this.update_();
+ // Draw separator bar
+ process.stdout.write(
+ `\x1B[36m` +
+ this.generateSeparator() +
+ `\x1B[0m\n`
+ );
+
+ // Redraw the static lines
+ this.static_lines.forEach(line => {
+ process.stdout.write(line + '\n');
+ });
+ process.stdout.write('\x1b[48;5;234m');
+ rl.resume();
+ rl._refreshLine();
+ process.stdout.write('\x1b[48;5;237m');
+ };
+
+ this._redraw = () => {
+ this._pre_write();
+ this._post_write();
+ };
+
+ setInterval(() => {
+ this._redraw();
+ }, 2000);
+
+ consoleLogManager.decorate_all(({ replace }, ...args) => {
+ this._pre_write();
+ });
+ consoleLogManager.post_all(() => {
+ this._post_write();
+ })
+ // logService.loggers.unshift({
+ // onLogMessage: () => {
+ // rl.pause();
+ // rl.output.write('\x1b[2K\r');
+ // }
+ // });
+ // logService.loggers.push({
+ // onLogMessage: () => {
+ // rl.resume();
+ // rl._refreshLine();
+ // }
+ // });
+
+ // This prevents the promptline background from staying
+ // when Ctrl+C is used to terminate the server
+ rl.on('SIGINT', () => {
+ process.stdout.write(`\x1b[0m\r`);
+ process.exit(0);
+ });
+ }
+
+ generateSeparator(text) {
+ text = text || '[ Dev Console ]';
+ const totalWidth = process.stdout.columns;
+ const paddingSize = (totalWidth - text.length) / 2;
+
+ // Construct the separator
+ return '═'.repeat(Math.floor(paddingSize)) + text + '═'.repeat(Math.ceil(paddingSize));
+ }
+
+ generateEnd(text) {
+ text = text || '';
+ const totalWidth = process.stdout.columns;
+ const paddingSize = (totalWidth - text.length) / 2;
+
+ // Construct the separator
+ return '─'.repeat(Math.floor(paddingSize)) + text + '─'.repeat(Math.ceil(paddingSize));
+ }
+}
+
+module.exports = {
+ DevConsoleService
+};
diff --git a/packages/backend/src/services/EmailService.js b/packages/backend/src/services/EmailService.js
new file mode 100644
index 00000000..37453d87
--- /dev/null
+++ b/packages/backend/src/services/EmailService.js
@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+class Emailservice extends AdvancedBase {
+ static MODULES = {
+ nodemailer: require('nodemailer'),
+ handlebars: require('handlebars'),
+ };
+
+ constructor ({ services, config }) {
+ super();
+ this.config = config;
+
+ this.templates = {
+ 'new-referral': {
+ subject: `You've made a referral!`,
+ html: `Hi there,
+ A new user has used your referral code. Enjoy an extra {{storage_increase}} of storage, on the house!
+ Sincerely,
+ Puter
+ `,
+ },
+ 'approved-for-listing': {
+ subject: '\u{1f389} Your app has been approved for listing!',
+ html: `
+Hi there,
+
+Exciting news! {{app_title}} is now approved and live on Puter App Center . It's now ready for users worldwide to discover and enjoy.
+
+
+Next Step : As your app begins to gain traction with more users, we will conduct periodic reviews to assess its performance and user engagement. Once your app meets our criteria, we'll invite you to our Incentive Program. This exclusive program will allow you to earn revenue each time users open your app. So, keep an eye out for updates and stay tuned for this exciting opportunity! Make sure to share your app with your fans, friends and family to help it gain traction: https://puter.com/app/{{app_name}}
+
+
+Best,
+The Puter Team
+
+ `,
+ },
+ };
+
+ this.template_fns = {};
+ for ( const k in this.templates ) {
+ const template = this.templates[k];
+ this.template_fns[k] = values => {
+ const html = this.modules.handlebars.compile(template.html);
+ return {
+ ...template,
+ html: html(values),
+ };
+ }
+ }
+ }
+
+ async send_email (user, template, values) {
+ const config = this.config;
+ const nodemailer = this.modules.nodemailer;
+
+ let transporter = nodemailer.createTransport({
+ host: config.smtp_server,
+ port: config.smpt_port,
+ secure: true, // STARTTLS
+ auth: {
+ user: config.smtp_username,
+ pass: config.smtp_password,
+ },
+ });
+
+ const email = user.email;
+
+ const template_fn = this.template_fns[template];
+ const { subject, html } = template_fn(values);
+
+ transporter.sendMail({
+ from: '"Puter" no-reply@puter.com', // sender address
+ to: email, // list of receivers
+ subject, html,
+ });
+ }
+}
+
+module.exports = {
+ Emailservice
+};
diff --git a/packages/backend/src/services/EngPortalService.js b/packages/backend/src/services/EngPortalService.js
new file mode 100644
index 00000000..7c146816
--- /dev/null
+++ b/packages/backend/src/services/EngPortalService.js
@@ -0,0 +1,162 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+class EngPortalService extends AdvancedBase {
+ static MODULES = {
+ socketio: require('../socketio.js'),
+ uuidv4: require('uuid').v4,
+ };
+
+ constructor ({ services }) {
+ super();
+ this.services = services;
+ this.commands = services.get('commands');
+ this._registerCommands(this.commands);
+ }
+
+ async list_operations () {
+ const svc_operationTrace = this.services.get('operationTrace');
+ const ls = [];
+ for ( const id in svc_operationTrace.ongoing ) {
+ const op = svc_operationTrace.ongoing[id];
+ ls.push(this._serialize_frame(op));
+ }
+
+ return ls;
+ }
+
+ _serialize_frame (frame) {
+ const out = {
+ id: frame.id,
+ label: frame.label,
+ status: frame.status,
+ async: frame.async,
+ checkpoint: frame.checkpoint,
+ // tags: frame.tags,
+ // attributes: frame.attributes,
+ // messages: frame.messages,
+ // error: frame.error_ ? frame.error_.message || true : null,
+ children: [],
+ attributes: {},
+ };
+
+ for ( const k in frame.attributes ) {
+ out.attributes[k] = frame.attributes[k];
+ }
+
+ for ( const child of frame.children ) {
+ out.children.push(this._serialize_frame(child));
+ }
+
+ return out;
+ }
+
+ async list_alarms () {
+ const svc_alarm = this.services.get('alarm');
+ const ls = [];
+ for ( const id in svc_alarm.alarms ) {
+ const alarm = svc_alarm.alarms[id];
+ ls.push(this._serialize_alarm(alarm));
+ }
+
+ return ls;
+ }
+
+ async get_stats () {
+ const svc_health = this.services.get('server-health');
+ return await svc_health.get_stats();
+ }
+
+ _serialize_alarm (alarm) {
+ const out = {
+ id: alarm.id,
+ short_id: alarm.short_id,
+ started: alarm.started,
+ occurrances: alarm.occurrences.map(this._serialize_occurance.bind(this)),
+ ...(alarm.error ? {
+ error: {
+ message: alarm.error.message,
+ stack: alarm.error.stack,
+ }
+ } : {}),
+ };
+
+ return out;
+ }
+
+ _serialize_occurance (occurance) {
+ const out = {
+ message: occurance.message,
+ timestamp: occurance.timestamp,
+ fields: occurance.fields,
+ };
+
+ return out;
+ }
+
+ _registerCommands (commands) {
+ this.commands.registerCommands('eng', [
+ {
+ id: 'test',
+ description: 'testing',
+ handler: async (args, log) => {
+ const ops = await this.list_operations();
+ log.log(JSON.stringify(ops, null, 2));
+ }
+ },
+ {
+ id: 'set',
+ description: 'set a parameter',
+ handler: async (args, log) => {
+ const [name, value] = args;
+ const parameter = this._get_param(name);
+ parameter.set(value);
+ log.log(value);
+ }
+ },
+ {
+ id: 'list',
+ description: 'list parameters',
+ handler: async (args, log) => {
+ const [prefix] = args;
+ let parameters = this.parameters_;
+ if ( prefix ) {
+ parameters = parameters
+ .filter(p => p.spec_.id.startsWith(prefix));
+ }
+ log.log(`available parameters${
+ prefix ? ` (starting with: ${prefix})` : ''
+ }:`);
+ for (const parameter of parameters) {
+ // log.log(`- ${parameter.spec_.id}: ${parameter.spec_.description}`);
+ // Log parameter description and value
+ const value = await parameter.get();
+ log.log(`- ${parameter.spec_.id} = ${value}`);
+ log.log(` ${parameter.spec_.description}`);
+ }
+ }
+ }
+ ]);
+ }
+}
+
+module.exports = {
+ EngPortalService,
+};
diff --git a/packages/backend/src/services/EntityStoreService.js b/packages/backend/src/services/EntityStoreService.js
new file mode 100644
index 00000000..e546262c
--- /dev/null
+++ b/packages/backend/src/services/EntityStoreService.js
@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../api/APIError");
+const { IdentifierUtil } = require("../om/IdentifierUtil");
+const { Null } = require("../om/query/query");
+const { Context } = require("../util/context");
+const BaseService = require("./BaseService");
+
+class EntityStoreService extends BaseService {
+ async _init (args) {
+ if ( ! args.entity ) {
+ throw new Error('EntityStoreService requires an entity name');
+ }
+
+ this.upstream = args.upstream;
+
+ const context = Context.get().sub({ services: this.services });
+ const om = this.services.get('registry').get('om:mapping').get(args.entity);
+ this.om = om;
+ await this.upstream.provide_context({
+ context,
+ om,
+ entity_name: args.entity,
+ });
+ }
+
+ // TODO: can replace these with MethodProxyTrait
+ async create (entity) {
+ return await this.upstream.upsert(entity, { old_entity: null });
+ }
+ async read (uid) {
+ return await this.upstream.read(uid);
+ }
+ async select ({ predicate, ...rest }) {
+ if ( ! predicate ) predicate = [];
+ if ( Array.isArray(predicate) ) {
+ const [p_op, ...p_args] = predicate;
+ predicate = await this.upstream.create_predicate(p_op, ...p_args);
+ }
+ if ( ! predicate) predicate = new Null();
+ return await this.upstream.select({ predicate, ...rest });
+ }
+ async update (entity, id) {
+ let old_entity = await this.read(
+ await entity.get(this.om.primary_identifier));
+
+ if ( ! old_entity ) {
+ const idu = new IdentifierUtil({
+ om: this.om,
+ });
+
+ const predicate = await idu.detect_identifier(id ?? {});
+ if ( predicate ) {
+ const maybe_entity = await this.select({ predicate, limit: 1 });
+ if ( maybe_entity.length ) {
+ old_entity = maybe_entity[0];
+ }
+ }
+ }
+
+ if ( ! old_entity ) {
+ throw APIError.create('entity_not_found', null, {
+ identifier: await entity.get(this.om.primary_identifier),
+ });
+ }
+
+ // Set primary identifier's value of `entity` to that in `old_entity`
+ const id_prop = this.om.properties[this.om.primary_identifier];
+ await entity.set(id_prop.name, await old_entity.get(id_prop.name));
+
+ return await this.upstream.upsert(entity, { old_entity });
+ }
+ async upsert (entity, id) {
+ let old_entity = await this.read(
+ await entity.get(this.om.primary_identifier));
+
+ if ( ! old_entity ) {
+ const idu = new IdentifierUtil({
+ om: this.om,
+ });
+
+ const predicate = await idu.detect_identifier(entity);
+ if ( predicate ) {
+ const maybe_entity = await this.select({ predicate, limit: 1 });
+ if ( maybe_entity.length ) {
+ old_entity = maybe_entity[0];
+ }
+ }
+ }
+
+ if ( old_entity ) {
+ // Set primary identifier's value of `entity` to that in `old_entity`
+ const id_prop = this.om.properties[this.om.primary_identifier];
+ await entity.set(id_prop.name, await old_entity.get(id_prop.name));
+ }
+
+ return await this.upstream.upsert(entity, { old_entity });
+ }
+ async delete (uid) {
+ const old_entity = await this.read(uid);
+ if ( ! old_entity ) {
+ throw APIError.create('entity_not_found', null, {
+ identifier: uid,
+ });
+ }
+ return await this.upstream.delete(uid, { old_entity });
+ }
+}
+
+module.exports = {
+ EntityStoreService,
+};
diff --git a/packages/backend/src/services/EventService.js b/packages/backend/src/services/EventService.js
new file mode 100644
index 00000000..0b07a08b
--- /dev/null
+++ b/packages/backend/src/services/EventService.js
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("./BaseService");
+
+class ScopedEventBus {
+ constructor (event_bus, scope) {
+ this.event_bus = event_bus;
+ this.scope = scope;
+ }
+
+ emit (key, data) {
+ this.event_bus.emit(this.scope + '.' + key, data);
+ }
+
+ on (key, callback) {
+ return this.event_bus.on(this.scope + '.' + key, callback);
+ }
+}
+
+class EventService extends BaseService {
+ async _construct () {
+ this.listeners_ = {};
+ }
+
+ emit (key, data) {
+ const parts = key.split('.');
+ for ( let i = 0; i < parts.length; i++ ) {
+ const part = i === parts.length - 1
+ ? parts.join('.')
+ : parts.slice(0, i + 1).join('.') + '.*';
+
+ // actual emit
+ const listeners = this.listeners_[part];
+ if ( ! listeners ) continue;
+ for ( let i = 0; i < listeners.length; i++ ) {
+ const callback = listeners[i];
+
+ // IIAFE wrapper to catch errors without blocking
+ // event dispatch.
+ (async () => {
+ try {
+ await callback(key, data);
+ } catch (e) {
+ this.errors.report('event-service.emit', {
+ source: e,
+ trace: true,
+ alarm: true,
+ });
+ }
+ })();
+ }
+ }
+
+ }
+
+ on (selector, callback) {
+ const listeners = this.listeners_[selector] ||
+ (this.listeners_[selector] = []);
+
+ listeners.push(callback);
+
+ const det = {
+ detach: () => {
+ const idx = listeners.indexOf(callback);
+ if ( idx !== -1 ) {
+ listeners.splice(idx, 1);
+ }
+ }
+ };
+
+ return det;
+ }
+
+ get_scoped (scope) {
+ return new ScopedEventBus(this, scope);
+ }
+}
+
+module.exports = {
+ EventService
+};
diff --git a/packages/backend/src/services/FilesystemAPIService.js b/packages/backend/src/services/FilesystemAPIService.js
new file mode 100644
index 00000000..fec3999b
--- /dev/null
+++ b/packages/backend/src/services/FilesystemAPIService.js
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("./BaseService");
+
+class FilesystemAPIService extends BaseService {
+ async ['__on_install.routes'] () {
+ const { app } = this.services.get('web-server');
+
+ // batch
+ app.use(require('../routers/filesystem_api/batch/all'))
+
+ // v2 -- also in batch
+ app.use(require('../routers/filesystem_api/write'))
+ app.use(require('../routers/filesystem_api/mkdir'))
+ app.use(require('../routers/filesystem_api/delete'))
+ // v2 -- not in batch
+ app.use(require('../routers/filesystem_api/stat'));
+ app.use(require('../routers/filesystem_api/touch'))
+ app.use(require('../routers/filesystem_api/read'))
+ app.use(require('../routers/filesystem_api/token-read'))
+ app.use(require('../routers/filesystem_api/readdir'))
+ app.use(require('../routers/filesystem_api/copy'))
+ app.use(require('../routers/filesystem_api/move'))
+ app.use(require('../routers/filesystem_api/rename'))
+
+ // v1
+ app.use(require('../routers/writeFile'))
+ app.use(require('../routers/file'))
+
+ // misc
+ app.use(require('../routers/df'))
+ app.use(require('../routers/download'))
+
+ }
+}
+
+module.exports = FilesystemAPIService;
diff --git a/packages/backend/src/services/LocalDiskStorageService.js b/packages/backend/src/services/LocalDiskStorageService.js
new file mode 100644
index 00000000..14b6d300
--- /dev/null
+++ b/packages/backend/src/services/LocalDiskStorageService.js
@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { LocalDiskStorageStrategy } = require("../filesystem/strategies/storage_a/LocalDiskStorageStrategy");
+const { TeePromise } = require("../util/promise");
+const { progress_stream } = require("../util/streamutil");
+const BaseService = require("./BaseService");
+
+class LocalDiskStorageService extends BaseService {
+ static MODULES = {
+ fs: require('fs'),
+ path: require('path'),
+ }
+
+ async ['__on_install.context-initializers'] () {
+ const svc_contextInit = this.services.get('context-init');
+ const storage = new LocalDiskStorageStrategy({ services: this.services });
+ svc_contextInit.register_value('storage', storage);
+ }
+
+ async _init () {
+ const require = this.require;
+ const path_ = require('path');
+
+ this.path = path_.join(process.cwd(), '/storage');
+
+ // ensure directory exists
+ const fs = require('fs');
+ await fs.promises.mkdir(this.path, { recursive: true });
+ }
+
+ _get_path (key) {
+ const require = this.require;
+ const path = require('path');
+ return path.join(this.path, key);
+ }
+
+ async store_stream ({ key, size, stream, on_progress }) {
+ const require = this.require;
+ const fs = require('fs');
+
+ stream = progress_stream(stream, {
+ total: size,
+ progress_callback: on_progress,
+ });
+
+ const writePromise = new TeePromise();
+
+ const path = this._get_path(key);
+ const write_stream = fs.createWriteStream(path);
+ write_stream.on('error', () => writePromise.reject());
+ write_stream.on('finish', () => writePromise.resolve());
+
+ stream.pipe(write_stream);
+
+ return await writePromise;
+ }
+
+ async store_buffer ({ key, buffer }) {
+ const require = this.require;
+ const fs = require('fs');
+
+ const path = this._get_path(key);
+ await fs.promises.writeFile(path, buffer);
+ }
+
+ async create_read_stream ({ key }) {
+ const require = this.require;
+ const fs = require('fs');
+
+ const path = this._get_path(key);
+ return fs.createReadStream(path);
+ }
+
+ async copy ({ src_key, dst_key }) {
+ const require = this.require;
+ const fs = require('fs');
+
+ const src_path = this._get_path(src_key);
+ const dst_path = this._get_path(dst_key);
+
+ await fs.promises.copyFile(src_path, dst_path);
+ }
+
+ async delete ({ key }) {
+ const require = this.require;
+ const fs = require('fs');
+
+ const path = this._get_path(key);
+ await fs.promises.unlink(path);
+ }
+}
+
+module.exports = LocalDiskStorageService;
diff --git a/packages/backend/src/services/MakeProdDebuggingLessAwfulService.js b/packages/backend/src/services/MakeProdDebuggingLessAwfulService.js
new file mode 100644
index 00000000..04295868
--- /dev/null
+++ b/packages/backend/src/services/MakeProdDebuggingLessAwfulService.js
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Context } = require("../util/context");
+const BaseService = require("./BaseService");
+const { stringify_log_entry } = require("./runtime-analysis/LogService");
+
+/**
+ * This service registers a middleware that will apply the value of
+ * header X-PUTER-DEBUG to the request's Context object.
+ *
+ * Consequentially, the value of X-PUTER-DEBUG will included in all
+ * log messages produced by the request.
+ */
+class MakeProdDebuggingLessAwfulService extends BaseService {
+ static MODULES = {
+ fs: require('fs'),
+ }
+ static ProdDebuggingMiddleware = class ProdDebuggingMiddleware {
+ constructor () {
+ this.header_name_ = 'x-puter-debug';
+ }
+ install (app) {
+ app.use(this.run.bind(this));
+ }
+ async run (req, res, next) {
+ const x = Context.get();
+ x.set('prod-debug', req.headers[this.header_name_]);
+ next();
+ }
+ }
+ async _init () {
+ // Initialize express middleware
+ this.mw = new this.constructor.ProdDebuggingMiddleware();
+
+ // Add logger middleware
+ const svc_log = this.services.get('log-service');
+ svc_log.register_log_middleware(async log_details => {
+ const {
+ context,
+ log_lvl, crumbs, message, fields, objects,
+ } = log_details;
+
+ const maybe_debug_token = context.get('prod-debug');
+
+ if ( ! maybe_debug_token ) return;
+
+ // Log to an additional log file so this is easier to find
+ const outfile = svc_log.get_log_file(`debug-${maybe_debug_token}.log`);
+
+ try {
+ await this.modules.fs.promises.appendFile(
+ outfile,
+ stringify_log_entry(log_details) + '\n',
+ );
+ } catch ( e ) {
+ console.error(e);
+ }
+
+ // Add the prod_debug field to the log message
+ return {
+ fields: {
+ ...fields,
+ prod_debug: maybe_debug_token,
+ }
+ };
+ });
+ }
+ async ['__on_install.middlewares.context-aware'] (_, { app }) {
+ // Add express middleware
+ this.mw.install(app);
+ }
+}
+
+module.exports = {
+ MakeProdDebuggingLessAwfulService,
+};
diff --git a/packages/backend/src/services/OperationTraceService.js b/packages/backend/src/services/OperationTraceService.js
new file mode 100644
index 00000000..fe039261
--- /dev/null
+++ b/packages/backend/src/services/OperationTraceService.js
@@ -0,0 +1,307 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { Context } = require("../util/context");
+const { ContextAwareTrait } = require("../traits/ContextAwareTrait");
+const { OtelTrait } = require("../traits/OtelTrait");
+const APIError = require("../api/APIError");
+const { AssignableMethodsTrait } = require("../traits/AssignableMethodsTrait");
+
+const CONTEXT_KEY = Context.make_context_key('operation-trace');
+
+class OperationFrame {
+ constructor ({ parent, label, x }) {
+ this.parent = parent;
+ this.label = label;
+ this.tags = [];
+ this.attributes = {};
+ this.messages = [];
+ this.error_ = null;
+ this.children = [];
+ this.status_ = this.constructor.FRAME_STATUS_PENDING;
+ this.effective_status_ = this.status_;
+ this.id = require('uuid').v4();
+
+ this.log = (x ?? Context).get('services').get('log-service').create(
+ `frame:${this.id}`
+ );
+ }
+
+ static FRAME_STATUS_PENDING = { label: 'pending' };
+ static FRAME_STATUS_WORKING = { label: 'working', };
+ static FRAME_STATUS_STUCK = { label: 'stuck' };
+ static FRAME_STATUS_READY = { label: 'ready' };
+ static FRAME_STATUS_DONE = { label: 'done' };
+
+ set status (status) {
+ this.status_ = status;
+ this._calc_effective_status();
+
+ this.log.info(
+ `FRAME STATUS ${status.label} ` +
+ (status !== this.effective_status_
+ ? `(effective: ${this.effective_status_.label}) `
+ : ''),
+ {
+ tags: this.tags,
+ ...this.attributes,
+ }
+ );
+
+ if ( this.parent ) {
+ this.parent._calc_effective_status();
+ }
+ }
+ _calc_effective_status () {
+ for ( const child of this.children ) {
+ if ( child.status === OperationFrame.FRAME_STATUS_STUCK ) {
+ this.effective_status_ = OperationFrame.FRAME_STATUS_STUCK;
+ return;
+ }
+ }
+
+ if ( this.status_ === OperationFrame.FRAME_STATUS_DONE ) {
+ for ( const child of this.children ) {
+ if ( child.status !== OperationFrame.FRAME_STATUS_DONE ) {
+ this.effective_status_ = OperationFrame.FRAME_STATUS_READY;
+ return;
+ }
+ }
+ }
+
+ this.effective_status_ = this.status_;
+ if ( this.parent ) {
+ this.parent._calc_effective_status();
+ }
+
+ // TODO: operation trace service should hook a listener instead
+ if ( this.effective_status_ === OperationFrame.FRAME_STATUS_DONE ) {
+ const svc_operationTrace = Context.get('services').get('operationTrace');
+ delete svc_operationTrace.ongoing[this.id];
+ }
+ }
+
+ get status () {
+ return this.effective_status_;
+ }
+
+ tag (...tags) {
+ this.tags.push(...tags);
+ return this;
+ }
+
+ attr (key, value) {
+ this.attributes[key] = value;
+ return this;
+ }
+
+ // recursively go through frames to find the attribute
+ get_attr (key) {
+ if ( this.attributes[key] ) return this.attributes[key];
+ if ( this.parent ) return this.parent.get_attr(key);
+ }
+
+ log (message) {
+ this.messages.push(message);
+ return this;
+ }
+
+ error (err) {
+ this.error_ = err;
+ return this;
+ }
+
+ push_child (frame) {
+ this.children.push(frame);
+ return this;
+ }
+
+ get_root_frame () {
+ let frame = this;
+ while ( frame.parent ) {
+ frame = frame.parent;
+ }
+ return frame;
+ }
+
+ done () {
+ this.status = OperationFrame.FRAME_STATUS_DONE;
+ }
+
+ describe (show_tree, highlight_frame) {
+ let s = this.label + ` (${this.children.length})`;
+ if ( this.tags.length ) {
+ s += ' ' + this.tags.join(' ');
+ }
+ if ( this.attributes ) {
+ s += ' ' + JSON.stringify(this.attributes);
+ }
+
+ if ( this.children.length == 0 ) return s;
+
+ // It's ASCII box drawing time!
+ const prefix_child = '├─';
+ const prefix_last = '└─';
+ const prefix_deep = '│ ';
+ const prefix_deep_end = ' ';
+
+ const recurse = (frame, prefix) => {
+ const children = frame.children;
+ for ( let i = 0; i < children.length; i++ ) {
+ const child = children[i];
+ const is_last = i == children.length - 1;
+ if ( child === highlight_frame ) s += `\x1B[36;1m`;
+ s += '\n' + prefix + (is_last ? prefix_last : prefix_child) + child.describe();
+ if ( child === highlight_frame ) s += `\x1B[0m`;
+ recurse(child, prefix + (is_last ? prefix_deep_end : prefix_deep));
+ }
+ }
+
+ if ( show_tree ) recurse(this, '');
+ return s;
+ }
+}
+
+class OperationTraceService {
+ constructor ({ services }) {
+ this.log = services.get('log-service').create('operation-trace');
+
+ // TODO: replace with kv.js set
+ this.ongoing = {};
+ }
+
+ async add_frame (label) {
+ return this.add_frame_sync(label);
+ }
+
+ add_frame_sync (label, x) {
+ if ( x ) {
+ this.log.noticeme(
+ 'add_frame_sync() called with explicit context: ' +
+ x.describe()
+ );
+ }
+ let parent = (x ?? Context).get(this.ckey('frame'));
+ const frame = new OperationFrame({
+ parent: parent || null,
+ label,
+ x
+ });
+ parent && parent.push_child(frame);
+ this.log.info(`FRAME START ` + frame.describe());
+ if ( ! parent ) {
+ // NOTE: only uncomment in local testing for now;
+ // this will cause a memory leak until frame
+ // done-ness is accurate
+ this.ongoing[frame.id] = frame;
+ }
+ return frame;
+ }
+
+ ckey (key) {
+ return CONTEXT_KEY + ':' + key;
+ }
+}
+
+class BaseOperation extends AdvancedBase {
+ static TRAITS = [
+ new ContextAwareTrait(),
+ new OtelTrait(['run']),
+ new AssignableMethodsTrait(),
+ ]
+
+ async run (values) {
+ this.values = values;
+
+ // getting context with a new operation frame
+ let x, frame; {
+ x = Context.get();
+ const operationTraceSvc = x.get('services').get('operationTrace');
+ frame = await operationTraceSvc.add_frame(this.constructor.name);
+ x = x.sub({ [operationTraceSvc.ckey('frame')]: frame });
+ }
+
+ // the frame will be an explicit property as well as being in context
+ // (for convenience)
+ this.frame = frame;
+
+ // let's make the logger for it too
+ this.log = x.get('services').get('log-service').create(
+ this.constructor.name, { operation: frame.id });
+
+ // Run operation in new context
+ try {
+ return await x.arun(async () => {
+ const x = Context.get();
+ const operationTraceSvc = x.get('services').get('operationTrace');
+ const frame = x.get(operationTraceSvc.ckey('frame'));
+ if ( ! frame ) {
+ throw new Error('missing frame');
+ }
+ frame.status = OperationFrame.FRAME_STATUS_WORKING;
+ this.checkpoint('._run()');
+ const res = await this._run();
+ this.checkpoint('._post_run()');
+ const { any_async } = this._post_run();
+ this.checkpoint('delegate .run_() returned');
+ frame.status = any_async
+ ? OperationFrame.FRAME_STATUS_READY
+ : OperationFrame.FRAME_STATUS_DONE;
+ return res;
+ });
+ } catch (e) {
+ if ( e instanceof APIError ) {
+ frame.attr('api-error', e.toString());
+ } else {
+ frame.error(e);
+ }
+ throw e;
+ }
+ }
+
+ checkpoint (name) {
+ this.frame.checkpoint = name;
+ }
+
+ field (key, value) {
+ this.frame.attributes[key] = value;
+ }
+
+ _post_run () {
+ let any_async = false;
+ for ( const child of this.frame.children ) {
+ if ( child.status === OperationFrame.FRAME_STATUS_PENDING ) {
+ child.status = OperationFrame.FRAME_STATUS_STUCK;
+ }
+
+ if ( child.status === OperationFrame.FRAME_STATUS_WORKING ) {
+ child.async = true;
+ any_async = true;
+ }
+ }
+ return { any_async };
+ }
+}
+
+module.exports = {
+ CONTEXT_KEY,
+ OperationTraceService,
+ BaseOperation,
+ OperationFrame,
+};
diff --git a/packages/backend/src/services/ParameterService.js b/packages/backend/src/services/ParameterService.js
new file mode 100644
index 00000000..65398d8b
--- /dev/null
+++ b/packages/backend/src/services/ParameterService.js
@@ -0,0 +1,157 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class ParameterService {
+ constructor({ services }) {
+ this.log = services.get('log-service').create('params');
+ this.parameters_ = [];
+
+ this._registerCommands(services.get('commands'));
+ }
+
+ createParameters(serviceName, parameters, opt_instance) {
+ for (const parameter of parameters) {
+ this.log.info(`registering parameter ${serviceName}:${parameter.id}`);
+ this.parameters_.push(new Parameter({
+ ...parameter,
+ id: `${serviceName}:${parameter.id}`,
+ }));
+ if ( opt_instance ) {
+ this.bindToInstance(
+ `${serviceName}:${parameter.id}`,
+ opt_instance,
+ parameter.id,
+ );
+ }
+ }
+ }
+
+ async get(id) {
+ const parameter = this._get_param(id);
+ return await parameter.get();
+ }
+
+ bindToInstance (id, instance, name) {
+ const parameter = this._get_param(id);
+ return parameter.bindToInstance(instance, name);
+ }
+
+ subscribe (id, listener) {
+ const parameter = this._get_param(id);
+ return parameter.subscribe(listener);
+ }
+
+ _get_param(id) {
+ const parameter = this.parameters_.find(p => p.spec_.id === id);
+ if ( ! parameter ) {
+ throw new Error(`unknown parameter: ${id}`);
+ }
+ return parameter;
+ }
+
+ _registerCommands (commands) {
+ commands.registerCommands('params', [
+ {
+ id: 'get',
+ description: 'get a parameter',
+ handler: async (args, log) => {
+ const [name] = args;
+ const value = await this.get(name);
+ log.log(value);
+ }
+ },
+ {
+ id: 'set',
+ description: 'set a parameter',
+ handler: async (args, log) => {
+ const [name, value] = args;
+ const parameter = this._get_param(name);
+ parameter.set(value);
+ log.log(value);
+ }
+ },
+ {
+ id: 'list',
+ description: 'list parameters',
+ handler: async (args, log) => {
+ const [prefix] = args;
+ let parameters = this.parameters_;
+ if ( prefix ) {
+ parameters = parameters
+ .filter(p => p.spec_.id.startsWith(prefix));
+ }
+ log.log(`available parameters${
+ prefix ? ` (starting with: ${prefix})` : ''
+ }:`);
+ for (const parameter of parameters) {
+ // log.log(`- ${parameter.spec_.id}: ${parameter.spec_.description}`);
+ // Log parameter description and value
+ const value = await parameter.get();
+ log.log(`- ${parameter.spec_.id} = ${value}`);
+ log.log(` ${parameter.spec_.description}`);
+ }
+ }
+ }
+ ]);
+ }
+}
+
+class Parameter {
+ constructor(spec) {
+ this.spec_ = spec;
+ this.valueListeners_ = [];
+
+ if ( spec.default ) {
+ this.value_ = spec.default;
+ }
+ }
+
+ async set (value) {
+ for ( const constraint of (this.spec_.constraints ?? []) ) {
+ if ( ! await constraint.check(value) ) {
+ throw new Error(`value ${value} does not satisfy constraint ${constraint.id}`);
+ }
+ }
+
+ const old = this.value_;
+ this.value_ = value;
+ for ( const listener of this.valueListeners_ ) {
+ listener(value, { old });
+ }
+ }
+
+ async get () {
+ return this.value_;
+ }
+
+ bindToInstance (instance, name) {
+ const value = this.value_;
+ instance[name] = value;
+ this.valueListeners_.push((value) => {
+ instance[name] = value;
+ });
+ }
+
+ subscribe (listener) {
+ this.valueListeners_.push(listener);
+ }
+}
+
+module.exports = {
+ ParameterService,
+};
diff --git a/packages/backend/src/services/PuterAPIService.js b/packages/backend/src/services/PuterAPIService.js
new file mode 100644
index 00000000..55c80712
--- /dev/null
+++ b/packages/backend/src/services/PuterAPIService.js
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("./BaseService");
+
+class PuterAPIService extends BaseService {
+ async ['__on_install.routes'] () {
+ const { app } = this.services.get('web-server');
+
+ app.use(require('../routers/version'))
+ app.use(require('../routers/apps'))
+ app.use(require('../routers/query/app'))
+ app.use(require('../routers/change_username'))
+ require('../routers/change_email')(app);
+ app.use(require('../routers/auth/get-user-app-token'))
+ app.use(require('../routers/auth/grant-user-app'))
+ app.use(require('../routers/auth/revoke-user-app'))
+ app.use(require('../routers/auth/list-permissions'))
+ app.use(require('../routers/auth/check-app'))
+ app.use(require('../routers/auth/app-uid-from-origin'))
+ app.use(require('../routers/auth/create-access-token'))
+ app.use(require('../routers/drivers/call'))
+ app.use(require('../routers/drivers/list-interfaces'))
+ app.use(require('../routers/drivers/usage'))
+ app.use(require('../routers/confirm-email'))
+ app.use(require('../routers/contactUs'))
+ app.use(require('../routers/delete-site'))
+ app.use(require('../routers/get-dev-profile'))
+ app.use(require('../routers/kvstore/getItem'))
+ app.use(require('../routers/kvstore/setItem'))
+ app.use(require('../routers/kvstore/listItems'))
+ app.use(require('../routers/kvstore/clearItems'))
+ app.use(require('../routers/get-launch-apps'))
+ app.use(require('../routers/itemMetadata'))
+ app.use(require('../routers/login'))
+ app.use(require('../routers/logout'))
+ app.use(require('../routers/open_item'))
+ app.use(require('../routers/passwd'))
+ app.use(require('../routers/rao'))
+ app.use(require('../routers/remove-site-dir'))
+ app.use(require('../routers/removeItem'))
+ app.use(require('../routers/save_account'))
+ app.use(require('../routers/send-confirm-email'))
+ app.use(require('../routers/send-pass-recovery-email'))
+ app.use(require('../routers/set-desktop-bg'))
+ app.use(require('../routers/set-pass-using-token'))
+ app.use(require('../routers/set_layout'))
+ app.use(require('../routers/set_sort_by'))
+ app.use(require('../routers/sign'))
+ app.use(require('../routers/signup'))
+ app.use(require('../routers/sites'))
+ // app.use(require('../routers/filesystem_api/stat'))
+ app.use(require('../routers/suggest_apps'))
+ app.use(require('../routers/test'))
+ app.use(require('../routers/update-taskbar-items'))
+ require('../routers/whoami')(app);
+
+ }
+}
+
+module.exports = PuterAPIService;
diff --git a/packages/backend/src/services/PuterSiteService.js b/packages/backend/src/services/PuterSiteService.js
new file mode 100644
index 00000000..444cda23
--- /dev/null
+++ b/packages/backend/src/services/PuterSiteService.js
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Context } = require("../util/context");
+const BaseService = require("./BaseService");
+const { DB_WRITE } = require("./database/consts");
+
+class PuterSiteService extends BaseService {
+ async _init () {
+ const services = this.services;
+ this.db = services.get('database').get(DB_WRITE, 'sites');
+ }
+
+ async get_subdomain (subdomain) {
+ if ( subdomain === 'devtest' && this.global_config.env === 'dev' ) {
+ return {
+ user_id: null,
+ root_dir_id: this.config.devtest_directory,
+ };
+ }
+ const rows = await this.read(
+ `SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1`,
+ [subdomain]
+ );
+ if ( rows.length === 0 ) return null;
+ return rows[0];
+ }
+}
+
+module.exports = {
+ PuterSiteService,
+};
diff --git a/packages/backend/src/services/PuterVersionService.js b/packages/backend/src/services/PuterVersionService.js
new file mode 100644
index 00000000..165812a1
--- /dev/null
+++ b/packages/backend/src/services/PuterVersionService.js
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { NodeUIDSelector } = require("../filesystem/node/selectors");
+const { get_user } = require("../helpers");
+const { TeePromise } = require("../util/promise");
+const { StreamBuffer } = require("../util/streamutil");
+
+class PuterVersionService extends AdvancedBase {
+ static MODULES = {
+ _path: require('path'),
+ fs: require('fs'),
+ _exec: require('child_process').execSync,
+ axios: require('axios'),
+ }
+
+ constructor ({ services, config }) {
+ super();
+ this.fs = services.get('filesystem');
+ this.config = config;
+
+ this._init({ config });
+ this.ready_ = new TeePromise();
+ }
+
+ async _init ({ config }) {
+ // this.node = await this.fs.node(new NodeUIDSelector(config.puter_hosted_data.puter_versions));
+ // this.user = await get_user({ username: 'puter' });
+
+ // await this._poll_versions({ config });
+
+ // setInterval(async () => {
+ // await this._poll_versions({ config });
+ // }, 60 * 1000);
+ }
+
+ // not used anymore - this was for getting version numbers from a file hosted on puter
+ async _poll_versions ({ config }) {
+ const resp = await this.modules.axios.get(
+ config.puter_hosted_data.puter_versions
+ );
+ this.versions_ = resp.data.versions;
+ this.ready_.resolve();
+ }
+
+ get_version () {
+ let git_version;
+ let deploy_timestamp;
+ if ( this.config.env === 'dev' ) {
+ // get commit hash from git
+ git_version = this.modules._exec('git describe --tags', {
+ cwd: this.modules._path.join(__dirname, '../../'),
+ encoding: 'utf8',
+ }).trim();
+ deploy_timestamp = Date.now();
+ } else {
+ // get git version from file
+ const path = this.modules._path.join(__dirname, '../../git_version');
+
+ git_version = this.modules.fs.readFileSync(path, 'utf8').trim();
+ deploy_timestamp = Number.parseInt((this.modules.fs.readFileSync(
+ this.modules._path.join(__dirname, '../../deploy_timestamp'),
+ 'utf8'
+ )).trim());
+ }
+ return {
+ version: git_version,
+ environment: this.config.env,
+ location: this.config.server_id,
+ deploy_timestamp,
+ };
+ }
+}
+
+module.exports = {
+ PuterVersionService,
+};
\ No newline at end of file
diff --git a/packages/backend/src/services/ReferralCodeService.js b/packages/backend/src/services/ReferralCodeService.js
new file mode 100644
index 00000000..07732cd9
--- /dev/null
+++ b/packages/backend/src/services/ReferralCodeService.js
@@ -0,0 +1,120 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const seedrandom = require('seedrandom');
+const { generate_random_code } = require('../util/identifier');
+const { Context } = require('../util/context');
+const { get_user } = require('../helpers');
+const { DB_WRITE } = require('./database/consts');
+
+class ReferralCodeService {
+ constructor ({ services }) {
+ this.log = services.get('log-service').create('referral-service');
+ this.errors = services.get('error-service').create(this.log);
+
+ this.REFERRAL_INCREASE_LEFT = 1 * 1024 * 1024 * 1024; // 1 GB
+ this.REFERRAL_INCREASE_RIGHT = 1 * 1024 * 1024 * 1024; // 1 GB
+ this.STORAGE_INCREASE_STRING = '1 GB';
+ }
+
+ async gen_referral_code (user) {
+ let iteration = 0;
+ let rng = seedrandom(`gen1-${user.id}`);
+ let referral_code = generate_random_code(8, { rng });
+
+ if ( ! user || (user?.id == undefined) ) {
+ const err = new Error('missing user in gen_referral_code');
+ this.errors.report('missing user in gen_referral_code', {
+ source: err,
+ trace: true,
+ alarm: true,
+ });
+ throw err;
+ }
+
+ const TRIES = 5;
+
+ const db = Context.get('services').get('database').get(DB_WRITE, 'referrals');
+
+ let last_error = null;
+ for ( let i=0 ; i < TRIES; i++ ) {
+ this.log.noticeme(`trying referral code ${referral_code}`)
+ if ( i > 0 ) {
+ rng = seedrandom(`gen1-${user.id}-${++iteration}`);
+ referral_code = generate_random_code(8, { rng });
+ }
+ try {
+ const update_res = db.write(`
+ UPDATE user SET referral_code=? WHERE id=?
+ `, [referral_code, user.id]);
+ return referral_code;
+ } catch (e) {
+ last_error = e;
+ }
+ }
+
+ this.errors.report('referral-service.gen-referral-code', {
+ source: last_error,
+ trace: true,
+ alarm: true,
+ });
+
+ throw last_error ?? new Error('unknown error from gen_referral_code');
+ }
+
+ async on_verified (user) {
+ if ( ! user.referred_by ) return;
+
+ const referred_by = await get_user({ id: user.referred_by });
+
+ // since this event handler is only called when the user is verified,
+ // we can assume that the `user` is already verified.
+
+ // the referred_by user does not need to be verified at all
+
+ // TODO: rename 'sizeService' to 'storage-capacity'
+ const svc_size = Context.get('services').get('sizeService');
+ await svc_size.add_storage(
+ user,
+ this.REFERRAL_INCREASE_RIGHT,
+ `user ${user.id} used referral code of user ${referred_by.id}`,
+ {
+ field_a: referred_by.referral_code,
+ field_b: 'REFER_R'
+ }
+ );
+ await svc_size.add_storage(
+ referred_by,
+ this.REFERRAL_INCREASE_LEFT,
+ `user ${referred_by.id} referred user ${user.id}`,
+ {
+ field_a: referred_by.referral_code,
+ field_b: 'REFER_L'
+ }
+ );
+
+ const svc_email = Context.get('services').get('email');
+ await svc_email.send_email (referred_by, 'new-referral', {
+ storage_increase: this.STORAGE_INCREASE_STRING
+ })
+ }
+}
+
+module.exports = {
+ ReferralCodeService
+};
diff --git a/packages/backend/src/services/RefreshAssociationsService.js b/packages/backend/src/services/RefreshAssociationsService.js
new file mode 100644
index 00000000..94c47ccb
--- /dev/null
+++ b/packages/backend/src/services/RefreshAssociationsService.js
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Context } = require("../util/context");
+const BaseService = require("./BaseService");
+
+class RefreshAssociationsService extends BaseService {
+ async ['__on_boot.services-initialized'] () {
+ const { refresh_associations_cache } = require('../helpers');
+
+ await Context.allow_fallback(async () => {
+ refresh_associations_cache();
+ });
+ setTimeout(() => {
+ setInterval(async () => {
+ await Context.allow_fallback(async () => {
+ await refresh_associations_cache();
+ })
+ }, 30000);
+ }, 15000)
+ }
+}
+
+module.exports = { RefreshAssociationsService };
diff --git a/packages/backend/src/services/RegistrantService.js b/packages/backend/src/services/RegistrantService.js
new file mode 100644
index 00000000..d8bba0a4
--- /dev/null
+++ b/packages/backend/src/services/RegistrantService.js
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Mapping } = require("../om/definitions/Mapping");
+const { PropType } = require("../om/definitions/PropType");
+const { Context } = require("../util/context");
+const BaseService = require("./BaseService");
+
+class RegistrantService extends BaseService {
+ async _init () {
+ const svc_systemValidation = this.services.get('system-validation');
+ try {
+ await this._populate_registry();
+ } catch ( e ) {
+ svc_systemValidation.mark_invalid(
+ 'Failed to populate registry',
+ e,
+ );
+ }
+ }
+ async _populate_registry () {
+ const svc_registry = this.services.get('registry');
+
+ // This context will be provided to the `create` methods
+ // that transform the raw data into objects.
+ const ctx = Context.get().sub({
+ registry: svc_registry,
+ });
+
+ // Register property types
+ {
+ const seen = new Set();
+
+ const collection = svc_registry.register_collection('om:proptype');
+ const data = require('../om/proptypes/__all__');
+ for ( const k in data ) {
+ if ( seen.has(k) ) {
+ throw new Error(`Duplicate property type "${k}"`);
+ }
+ if ( data[k].from && ! seen.has(data[k].from) ) {
+ throw new Error(`Super type "${data[k].from}" not found for property type "${k}"`);
+ }
+ collection.set(k, PropType.create(ctx, data[k]));
+ seen.add(k);
+ }
+ }
+
+ // Register object mappings
+ {
+ const collection = svc_registry.register_collection('om:mapping');
+ const data = require('../om/mappings/__all__');
+ for ( const k in data ) {
+ collection.set(k, Mapping.create(ctx, data[k]));
+ }
+ }
+ }
+}
+
+module.exports = {
+ RegistrantService,
+};
diff --git a/packages/backend/src/services/RegistryService.js b/packages/backend/src/services/RegistryService.js
new file mode 100644
index 00000000..848e9961
--- /dev/null
+++ b/packages/backend/src/services/RegistryService.js
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const BaseService = require("./BaseService");
+
+class MapCollection extends AdvancedBase {
+ static MODULES = {
+ kv: globalThis.kv,
+ uuidv4: require('uuid').v4,
+ }
+ constructor () {
+ super();
+ // We use kvjs instead of a plain object because it doesn't
+ // have a limit on the number of keys it can store.
+ this.map_id = this.modules.uuidv4();
+ this.kv = kv;
+ }
+
+ get (key) {
+ return this.kv.get(this._mk_key(key));
+ }
+
+ set (key, value) {
+ return this.kv.set(this._mk_key(key), value);
+ }
+
+ del (key) {
+ return this.kv.del(this._mk_key(key));
+ }
+
+ _mk_key (key) {
+ return `registry:map:${this.map_id}:${key}`;
+ }
+}
+
+class RegistryService extends BaseService {
+ static MODULES = {
+ MapCollection,
+ }
+
+ _construct () {
+ this.collections_ = {};
+ }
+
+ register_collection (name) {
+ if ( this.collections_[name] ) {
+ throw Error(`collection ${name} already exists`);
+ }
+ this.collections_[name] = new this.modules.MapCollection();
+ return this.collections_[name];
+ }
+
+ get (name) {
+ if ( ! this.collections_[name] ) {
+ throw Error(`collection ${name} does not exist`);
+ }
+ return this.collections_[name];
+ }
+}
+
+module.exports = {
+ RegistryService,
+};
diff --git a/packages/backend/src/services/ServeGUIService.js b/packages/backend/src/services/ServeGUIService.js
new file mode 100644
index 00000000..e378a0ec
--- /dev/null
+++ b/packages/backend/src/services/ServeGUIService.js
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("./BaseService");
+
+const express = require('express');
+const _path = require('path');
+
+class ServeGUIService extends BaseService {
+ async ['__on_install.routes-gui'] () {
+ const { app } = this.services.get('web-server');
+
+ // Router for all other cases
+ app.use(require('../routers/_default'))
+
+ // Static files
+ app.use(express.static(_path.join(__dirname, '../../public')))
+
+ // is this a puter.site domain?
+ require('../routers/hosting/puter-site')(app);
+ }
+}
+
+module.exports = ServeGUIService;
+
diff --git a/packages/backend/src/services/StorageService.js b/packages/backend/src/services/StorageService.js
new file mode 100644
index 00000000..bbf97b3a
--- /dev/null
+++ b/packages/backend/src/services/StorageService.js
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+class StorageService extends AdvancedBase {
+ constructor ({ services }) {
+ //
+ }
+}
\ No newline at end of file
diff --git a/packages/backend/src/services/StrategizedService.js b/packages/backend/src/services/StrategizedService.js
new file mode 100644
index 00000000..60664885
--- /dev/null
+++ b/packages/backend/src/services/StrategizedService.js
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { TechnicalError } = require("../errors/TechnicalError");
+const { quot } = require("../util/strutil");
+
+class StrategizedService {
+ constructor (service_resources, ...a) {
+ const { my_config, args, name } = service_resources;
+
+ const key = args.strategy_key;
+ if ( ! args.default_strategy && ! my_config.hasOwnProperty(key) ) {
+ this.initError = new TechnicalError(
+ `Must specify ${quot(key)} for service ${quot(name)}.`
+ );
+ return;
+ }
+
+ if ( ! args.hasOwnProperty('strategies') ) {
+ throw new Error('strategies not defined in service args')
+ }
+
+ const strategy_key = my_config[key] ?? args.default_strategy;
+ if ( ! args.strategies.hasOwnProperty(strategy_key) ) {
+ this.initError = new TechnicalError(
+ `Invalid ${key} ${quot(strategy_key)} for service ${quot(name)}.`
+ );
+ return;
+ }
+ const [cls, cls_args] = args.strategies[strategy_key];
+
+ const cls_resources = {
+ ...service_resources,
+ args: cls_args,
+ };
+ this.strategy = new cls(cls_resources, ...a);
+
+ return this.strategy;
+ }
+
+ async init () {
+ throw this.initError;
+ }
+
+ async construct () {}
+}
+
+module.exports = {
+ StrategizedService,
+};
diff --git a/packages/backend/src/services/SystemValidationService.js b/packages/backend/src/services/SystemValidationService.js
new file mode 100644
index 00000000..f5bdc857
--- /dev/null
+++ b/packages/backend/src/services/SystemValidationService.js
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("./BaseService");
+
+class SystemValidationService extends BaseService {
+ /**
+ * Marks the server is being in an invalid state.
+ *
+ * This is a very serious error. The server will do whatever it can to get
+ * our attention, and then it will shut down after 25 minutes.
+ *
+ * @param {*} message - why mark_invalid was called
+ * @param {*} source - the error that caused the invalid state, if any
+ */
+ async mark_invalid (message, source) {
+ if ( ! source ) source = new Error('no source error');
+
+ // The system is in an invalid state. The server will do whatever it
+ // can to get our attention, and then it will shut down.
+ if ( ! this.errors ) {
+ console.error(
+ 'SystemValidationService is trying to mark the system as invalid, but the error service is not available.',
+ message,
+ source,
+ );
+
+ // We can't do anything else. The server will crash.
+ throw new Error('SystemValidationService is trying to mark the system as invalid, but the error service is not available.');
+ }
+
+ this.errors.report('INVALID SYSTEM STATE', {
+ source,
+ message,
+ trace: true,
+ alarm: true,
+ });
+
+ // If we're in dev mode...
+ if ( this.global_config.env === 'dev' ) {
+ // Display a permanent message in the console
+ const svc_devConsole = this.services.get('dev-console');
+ svc_devConsole.turn_on_the_warning_lights();
+ svc_devConsole.add_widget(() => {
+ return `\x1B[33;1m *** SYSTEM IS IN AN INVALID STATE *** \x1B[0m`;
+ });
+
+ // Don't shut down
+ return;
+ }
+
+ // Raise further alarms if the system keeps running
+ for ( let i = 0; i < 5; i++ ) {
+ // After 5 minutes, raise another alarm
+ await new Promise(rslv => setTimeout(rslv, 60 * 5000));
+ this.errors.report(`INVALID SYSTEM STATE (Reminder ${i+1})`, {
+ source,
+ message,
+ trace: true,
+ alarm: true,
+ });
+ }
+ }
+}
+
+module.exports = { SystemValidationService };
diff --git a/packages/backend/src/services/TraceService.js b/packages/backend/src/services/TraceService.js
new file mode 100644
index 00000000..de156495
--- /dev/null
+++ b/packages/backend/src/services/TraceService.js
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const opentelemetry = require("@opentelemetry/api");
+
+class TraceService {
+ constructor () {
+ this.tracer_ = opentelemetry.trace.getTracer(
+ 'puter-filesystem-tracer'
+ );
+ }
+
+ get tracer () {
+ return this.tracer_;
+ }
+
+ async spanify (name, fn) {
+ return await this.tracer.startActiveSpan(name, async span => {
+ try {
+ return await fn({ span });
+ } catch (error) {
+ span.setStatus({ code: opentelemetry.SpanStatusCode.ERROR, message: error.message });
+ throw error;
+ } finally {
+ span.end();
+ }
+ });
+ }
+}
+
+module.exports = {
+ TraceService,
+};
diff --git a/packages/backend/src/services/TrackSpendingService.js b/packages/backend/src/services/TrackSpendingService.js
new file mode 100644
index 00000000..9fb4bc66
--- /dev/null
+++ b/packages/backend/src/services/TrackSpendingService.js
@@ -0,0 +1,243 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { TimeWindow } = require("../util/opmath");
+const SmolUtil = require("../util/smolutil");
+const { format_as_usd } = require("../util/strutil");
+const { MINUTE, SECOND } = require("../util/time");
+const BaseService = require("./BaseService");
+
+class TrackSpendingService extends BaseService {
+ static ChatCompletionStrategy = class ChatCompletionStrategy {
+ static models = {
+ 'gpt-4-1106-preview': {
+ cost_per_input_token: [0.01, 1000],
+ cost_per_output_token: [0.03, 1000],
+ },
+ 'gpt-4-vision-preview': {
+ cost_per_input_token: [0.01, 1000],
+ cost_per_output_token: [0.03, 1000],
+ },
+ 'gpt-3.5-turbo': {
+ cost_per_input_token: [0.001, 1000],
+ cost_per_output_token: [0.002, 1000],
+ },
+ };
+ constructor ({ service }) {
+ this.service = service;
+ }
+
+ multiply_by_ratio_ (value, [numerator, denominator]) {
+ return value * numerator / denominator;
+ }
+
+ get_cost (vendor, data) {
+ const model = data.model ?? 'gpt-4-1106-preview';
+ const model_pricing = this.constructor.models[model];
+
+ if ( ! model_pricing ) {
+ throw new Error(`unknown model ${model}`);
+ }
+
+ const cost_per_input_token = model_pricing.cost_per_input_token;
+ const cost_per_output_token = model_pricing.cost_per_output_token;
+
+ const input_tokens = data.count_tokens_input ?? 0;
+ const output_tokens = data.count_tokens_output ?? 0;
+
+ const cost = SmolUtil.add(
+ this.multiply_by_ratio_(input_tokens, cost_per_input_token),
+ this.multiply_by_ratio_(output_tokens, cost_per_output_token),
+ );
+
+ console.log('COST IS', cost);
+
+ return cost;
+ }
+
+ async validate () {
+ // Ensure no models will cause division by zero
+ for ( const model in this.constructor.models ) {
+ const model_pricing = this.constructor.models[model];
+ if ( model_pricing.cost_per_input_token[1] === 0 ) {
+ throw new Error(`model ${model} pricing conf (input tokens) will cause division by zero`);
+ }
+ if ( model_pricing.cost_per_output_token[1] === 0 ) {
+ throw new Error(`model ${model} pricing conf (output tokens) will cause division by zero`);
+ }
+ }
+ }
+ }
+ static ImageGenerationStrategy = class ImageGenerationStrategy {
+ static models = {
+ 'dall-e-3': {
+ '1024x1024': 0.04,
+ '1024x1792': 0.08,
+ '1792x1024': 0.08,
+ 'hd:1024x1024': 0.08,
+ 'hd:1024x1792': 0.12,
+ 'hd:1792x1024': 0.12,
+ },
+ 'dall-e-2': {
+ '1024x1024': 0.02,
+ '512x512': 0.018,
+ '256x256': 0.016,
+ },
+ };
+ constructor ({ service }) {
+ this.service = service;
+ }
+
+ multiply_by_ratio_ (value, [numerator, denominator]) {
+ return value * numerator / denominator;
+ }
+
+ get_cost (vendor, data) {
+ const model = data.model ?? 'dall-e-2';
+ const model_pricing = this.constructor.models[model];
+
+ if ( ! model_pricing ) {
+ throw new Error(`unknown model ${model}`);
+ }
+
+ if ( ! model_pricing.hasOwnProperty(data.size) ) {
+ throw new Error(`unknown size ${data.size} for model ${model}`);
+ }
+
+ const cost = model_pricing[data.size];
+
+ console.log('COST IS', cost);
+
+ return cost;
+ }
+ }
+
+ async _init () {
+ const strategies = {
+ 'chat-completion': new this.constructor.ChatCompletionStrategy({
+ service: this,
+ }),
+ 'image-generation': new this.constructor.ImageGenerationStrategy({
+ service: this,
+ }),
+ };
+
+ // How quickly we get the first alarm
+ const alarm_check_interval = 10 * SECOND;
+
+ // How frequently we'll get repeat alarms
+ const alarm_cooldown_time = 30 * MINUTE;
+
+ const alarm_at_cost = this.config.alarm_at_cost ?? 1;
+ const alarm_increment = this.config.alarm_increment ?? 1;
+
+ for ( const k in strategies ) {
+ await strategies[k].validate?.();
+ }
+
+ if ( ! this.log ) {
+ throw new Error('no log?');
+ }
+
+ this.strategies = strategies;
+
+ // Tracks overall server spending
+ this.spend_windows = {};
+
+ // Tracks what dollar amounts alerts were reported for
+ this.alerts_window = new TimeWindow({
+ // window_duration: 30 * MINUTE,
+ window_duration: alarm_cooldown_time,
+ reducer: a => Math.max(0, ...a),
+ });
+
+ const svc_alarm = this.services.get('alarm');
+
+ setInterval(() => {
+ const spending = this.get_window_spending_();
+
+ const increment = Math.floor(spending / alarm_increment);
+ const last_increment = this.alerts_window.get();
+
+ if ( increment <= last_increment ) {
+ return;
+ }
+
+ this.log.info('adding that increment');
+ this.alerts_window.add(increment);
+
+ if ( spending >= alarm_at_cost ) {
+ // see: src/polyfill/to-string-higher-radix.js
+ const ts_for_id = Date.now().toString(62);
+
+ this.log.info('triggering alarm');
+ this.log.info('alarm at: ' + alarm_at_cost);
+ this.log.info('spend: ' + this.get_window_spending_());
+ svc_alarm.create(
+ `high-spending-${ts_for_id}`,
+ `server spending is ${spending} within 30 minutes`,
+ {
+ spending,
+ increment_level: increment,
+ },
+ );
+ }
+ }, alarm_check_interval);
+ }
+
+ add_or_get_window_ (id) {
+ if ( this.spend_windows[id] ) {
+ return this.spend_windows[id];
+ }
+
+ return this.spend_windows[id] = new TimeWindow({
+ // window_duration: 30 * MINUTE,
+ window_duration: 30 * MINUTE,
+ reducer: a => a.reduce((a, b) => a + b, 0),
+ });
+ }
+
+ get_window_spending_ () {
+ const windows = Object.values(this.spend_windows);
+ return windows.reduce((sum, win) => {
+ return sum + win.get();
+ }, 0);
+ }
+
+ record_spending (vendor, strategy_key, data) {
+ const strategy = this.strategies[strategy_key];
+ if ( ! strategy ) {
+ throw new Error(`unknown strategy ${strategy_key}`);
+ }
+
+ const cost = strategy.get_cost(vendor, data);
+
+ this.log.info(`Spent ${format_as_usd(cost)}`, {
+ vendor, strategy_key, data,
+ cost,
+ })
+
+ const id = `${vendor}:${strategy_key}`;
+ const window = this.add_or_get_window_(id);
+ window.add(cost);
+ }
+}
+
+module.exports = {
+ TrackSpendingService,
+};
diff --git a/packages/backend/src/services/WSPushService.js b/packages/backend/src/services/WSPushService.js
new file mode 100644
index 00000000..eed2a6cc
--- /dev/null
+++ b/packages/backend/src/services/WSPushService.js
@@ -0,0 +1,232 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+class WSPushService extends AdvancedBase {
+ static MODULES = {
+ socketio: require('../socketio.js'),
+ }
+
+ constructor ({ services }) {
+ super();
+ this.log = services.get('log-service').create('WSPushService');
+ this.svc_event = services.get('event');
+
+ this.svc_event.on('fs.create.*', this._on_fs_create.bind(this));
+ this.svc_event.on('fs.write.*', this._on_fs_update.bind(this));
+ this.svc_event.on('fs.move.*', this._on_fs_move.bind(this));
+ this.svc_event.on('fs.pending.*', this._on_fs_pending.bind(this));
+ this.svc_event.on('fs.storage.upload-progress',
+ this._on_upload_progress.bind(this));
+ this.svc_event.on('fs.storage.progress.*',
+ this._on_upload_progress.bind(this));
+ }
+
+ async _on_fs_create (key, data) {
+ const { node, context } = data;
+ const { socketio } = this.modules;
+
+ const metadata = {
+ from_new_service: true,
+ };
+
+ {
+ const svc_operationTrace = context.get('services').get('operationTrace');
+ const frame = context.get(svc_operationTrace.ckey('frame'));
+ const gui_metadata = frame.get_attr('gui_metadata') || {};
+ Object.assign(metadata, gui_metadata);
+ }
+
+ const response = await node.getSafeEntry({ thumbnail: true });
+
+ const user_id_list = await (async () => {
+ // NOTE: Using a set because eventually we will need to dispatch
+ // to multiple users, but this is not currently the case.
+ const user_id_set = new Set();
+ if ( metadata.user_id ) user_id_set.add(metadata.user_id);
+ else user_id_set.add(await node.get('user_id'));
+ return Array.from(user_id_set);
+ })();
+
+ Object.assign(response, metadata);
+
+ const io = socketio.getio();
+ for ( const user_id of user_id_list ) {
+ io.to(user_id).emit('item.added', response);
+ }
+ }
+
+ async _on_fs_update (key, data) {
+ const { node, context } = data;
+ const { socketio } = this.modules;
+
+ const metadata = {
+ from_new_service: true,
+ };
+
+ {
+ const svc_operationTrace = context.get('services').get('operationTrace');
+ const frame = context.get(svc_operationTrace.ckey('frame'));
+ const gui_metadata = frame.get_attr('gui_metadata') || {};
+ Object.assign(metadata, gui_metadata);
+ }
+
+ const response = await node.getSafeEntry({ debug: 'hi', thumbnail: true });
+
+ const user_id_list = await (async () => {
+ // NOTE: Using a set because eventually we will need to dispatch
+ // to multiple users, but this is not currently the case.
+ const user_id_set = new Set();
+ if ( metadata.user_id ) user_id_set.add(metadata.user_id);
+ else user_id_set.add(await node.get('user_id'));
+ return Array.from(user_id_set);
+ })();
+
+ Object.assign(response, metadata);
+
+ const io = socketio.getio();
+ for ( const user_id of user_id_list ) {
+ io.to(user_id).emit('item.updated', response);
+ }
+ }
+
+ async _on_fs_move (key, data) {
+ const { moved, old_path, context } = data;
+ const { socketio } = this.modules;
+
+ const metadata = {
+ from_new_service: true,
+ };
+
+ {
+ const svc_operationTrace = context.get('services').get('operationTrace');
+ const frame = context.get(svc_operationTrace.ckey('frame'));
+ const gui_metadata = frame.get_attr('gui_metadata') || {};
+ Object.assign(metadata, gui_metadata);
+ }
+
+ const response = await moved.getSafeEntry();
+
+ const user_id_list = await (async () => {
+ // NOTE: Using a set because eventually we will need to dispatch
+ // to multiple users, but this is not currently the case.
+ const user_id_set = new Set();
+ if ( metadata.user_id ) user_id_set.add(metadata.user_id);
+ else user_id_set.add(await moved.get('user_id'));
+ return Array.from(user_id_set);
+ })();
+
+ response.old_path = old_path;
+ Object.assign(response, metadata);
+
+ const io = socketio.getio();
+ for ( const user_id of user_id_list ) {
+ io.to(user_id).emit('item.moved', response);
+ }
+ }
+
+ async _on_fs_pending (key, data) {
+ const { fsentry, context } = data;
+ const { socketio } = this.modules;
+
+ const metadata = {
+ from_new_service: true,
+ };
+
+ const response = { ...fsentry };
+
+ {
+ const svc_operationTrace = context.get('services').get('operationTrace');
+ const frame = context.get(svc_operationTrace.ckey('frame'));
+ const gui_metadata = frame.get_attr('gui_metadata') || {};
+ Object.assign(metadata, gui_metadata);
+ }
+
+ const user_id_list = await (async () => {
+ // NOTE: Using a set because eventually we will need to dispatch
+ // to multiple users, but this is not currently the case.
+ const user_id_set = new Set();
+ if ( metadata.user_id ) user_id_set.add(metadata.user_id);
+ return Array.from(user_id_set);
+ })();
+
+ Object.assign(response, metadata);
+
+ const io = socketio.getio();
+ for ( const user_id of user_id_list ) {
+ io.to(user_id).emit('item.pending', response);
+ }
+ }
+
+ async _on_upload_progress (key, data) {
+ this.log.info('got upload progress event');
+ const { socketio } = this.modules;
+ const { upload_tracker, context, meta } = data;
+
+ const metadata = {
+ ...meta,
+ from_new_service: true,
+ };
+
+ {
+ const svc_operationTrace = context.get('services').get('operationTrace');
+ const frame = context.get(svc_operationTrace.ckey('frame'));
+ const gui_metadata = frame.get_attr('gui_metadata') || {};
+ Object.assign(metadata, gui_metadata);
+ }
+
+ const { socket_id } = metadata;
+
+ if ( ! socket_id ) {
+ this.log.error('missing socket id', { metadata });
+
+ // TODO: this error is temporarily disabled for
+ // Puter V1 release, because it will cause a
+ // lot of redundant PagerDuty alerts.
+
+ // throw new Error('missing socket id');
+ }
+
+ this.log.info('socket id: ' + socket_id);
+
+ const io = socketio.getio()
+ .sockets.sockets
+ .get(socket_id);
+
+ // socket disconnected; that's allowed
+ if ( ! io ) return;
+
+ const ws_event_name = metadata.call_it_download
+ ? 'download.progress' : 'upload.progress' ;
+
+ upload_tracker.sub(delta => {
+ this.log.info('emitting progress event');
+ io.emit(ws_event_name, {
+ ...metadata,
+ total: upload_tracker.total_,
+ loaded: upload_tracker.progress_,
+ loaded_diff: delta,
+ })
+ })
+ }
+}
+
+module.exports = {
+ WSPushService
+};
diff --git a/packages/backend/src/services/WebServerService.js b/packages/backend/src/services/WebServerService.js
new file mode 100644
index 00000000..15bffd37
--- /dev/null
+++ b/packages/backend/src/services/WebServerService.js
@@ -0,0 +1,366 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const express = require('express');
+const eggspress = require("../api/eggspress");
+const { Context, ContextExpressMiddleware } = require("../util/context");
+const BaseService = require("./BaseService");
+
+const config = require('../config');
+const https = require('https')
+var http = require('http');
+const fs = require('fs');
+const auth = require('../middleware/auth');
+const { osclink } = require('../util/strutil');
+
+class WebServerService extends BaseService {
+ static MODULES = {
+ https: require('https'),
+ http: require('http'),
+ fs: require('fs'),
+ express: require('express'),
+ helmet: require('helmet'),
+ cookieParser: require('cookie-parser'),
+ compression: require('compression'),
+ ['on-finished']: require('on-finished'),
+ morgan: require('morgan'),
+ };
+
+ async ['__on_start.webserver'] () {
+
+ // error handling middleware goes last, as per the
+ // expressjs documentation:
+ // https://expressjs.com/en/guide/error-handling.html
+ this.app.use(require('../api/api_error_handler'));
+
+ const path = require('path')
+ const { jwt_auth } = require('../helpers');
+
+ config.http_port = process.env.PORT ?? config.http_port;
+
+ globalThis.deployment_type =
+ config.http_port === 5101 ? 'green' :
+ config.http_port === 5102 ? 'blue' :
+ 'not production';
+
+ let server;
+
+ const auto_port = config.http_port === 'auto';
+ let ports_to_try = auto_port ? (() => {
+ const ports = [];
+ for ( let i = 0 ; i < 20 ; i++ ) {
+ ports.push(4100 + i);
+ }
+ return ports;
+ })() : [Number.parseInt(config.http_port)];
+
+ for ( let i = 0 ; i < ports_to_try.length ; i++ ) {
+ const port = ports_to_try[i];
+ const is_last_port = i === ports_to_try.length - 1;
+ if ( auto_port ) this.log.info('trying port: ' + port);
+ try {
+ console.log('---AA--- ', is_last_port);
+ console.log('---BB--- ', ports_to_try);
+ server = http.createServer(this.app).listen(port);
+ server.timeout = 1000 * 60 * 60 * 2; // 2 hours
+ let should_continue = false;
+ await new Promise((rslv, rjct) => {
+ server.on('error', e => {
+ if ( e.code === 'EADDRINUSE' ) {
+ if ( ! is_last_port && e.code === 'EADDRINUSE' ) {
+ this.log.info('port in use:', port);
+ should_continue = true;
+ }
+ rjct(e);
+ }
+ rslv();
+ });
+ server.on('listening', () => {
+ rslv();
+ })
+ })
+ if ( should_continue ) continue;
+ } catch (e) {
+ console.log('CONDITION?', e.code, is_last_port)
+ if ( ! is_last_port && e.code === 'EADDRINUSE' ) {
+ this.log.info('port in use:', port);
+ continue;
+ }
+ throw e;
+ }
+ config.http_port = port;
+ break;
+ }
+ ports_to_try = null; // GC
+
+ const url = config.origin;
+
+ this.startup_widget = () => {
+ const link = `\x1B[34;1m${osclink(url)}\x1B[0m`;
+ const lines = [
+ "",
+ `Puter is now live at: ${link}`,
+ `Type web:dismiss to dismiss this message`,
+ "",
+ ];
+ const lengths = [
+ 0,
+ (`Puter is now live at: `).length + url.length,
+ lines[2].length,
+ 0,
+ ];
+ const max_length = Math.max(...lengths);
+ const c = str => `\x1b[34;1m${str}\x1b[0m`;
+ const bar = c(Array(max_length + 4).fill('━').join(''));
+ for ( let i = 0 ; i < lines.length ; i++ ) {
+ while ( lines[i].length < max_length ) {
+ lines[i] += ' ';
+ }
+ lines[i] = `${c('┃ ')} ${lines[i]} ${c(' ┃')}`;
+ }
+ lines.unshift(`${c('┏')}${bar}${c('┓')}`);
+ lines.push(`${c('┗')}${bar}${c('┛')}`);
+ return lines;
+ };
+ {
+ const svc_devConsole = this.services.get('dev-console');
+ svc_devConsole.add_widget(this.startup_widget);
+ }
+
+ this.print_puter_logo_();
+
+ server.timeout = 1000 * 60 * 60 * 2; // 2 hours
+ server.requestTimeout = 1000 * 60 * 60 * 2; // 2 hours
+ server.headersTimeout = 1000 * 60 * 60 * 2; // 2 hours
+ // server.keepAliveTimeout = 1000 * 60 * 60 * 2; // 2 hours
+
+ // Socket.io server instance
+ const socketio = require('../socketio.js').init(server);
+
+ // Socket.io middleware for authentication
+ socketio.use(async (socket, next) => {
+ if (socket.handshake.query.auth_token) {
+ try {
+ let auth_res = await jwt_auth(socket);
+ // successful auth
+ socket.user = auth_res.user;
+ socket.token = auth_res.token;
+ // join user room
+ socket.join(socket.user.id);
+ next();
+ } catch (e) {
+ console.log('socket auth err');
+ }
+ }
+ });
+
+ socketio.on('connection', (socket) => {
+ socket.on('disconnect', () => {
+ });
+ socket.on('trash.is_empty', (msg) => {
+ socket.broadcast.to(socket.user.id).emit('trash.is_empty', msg);
+ });
+ });
+ }
+
+ async _init () {
+ const app = express();
+ this.app = app;
+
+ app.set('services', this.services);
+ this._register_commands(this.services.get('commands'));
+
+ this.middlewares = { auth };
+
+
+ const require = this.require;
+
+ const config = this.global_config;
+ new ContextExpressMiddleware({
+ parent: globalThis.root_context.sub({
+ puter_environment: Context.create({
+ env: config.env,
+ version: require('../../package.json').version,
+ }),
+ }, 'mw')
+ }).install(app);
+
+ app.use(async (req, res, next) => {
+ req.services = this.services;
+ next();
+ });
+
+ // Instrument logging to use our log service
+ {
+ const morgan = require('morgan');
+ const stream = {
+ write: (message) => {
+ const [method, url, status, responseTime] = message.split(' ')
+ const fields = {
+ method,
+ url,
+ status: parseInt(status, 10),
+ responseTime: parseFloat(responseTime),
+ };
+ if ( url.includes('android-icon') ) return;
+ const log = this.services.get('log-service').create('morgan');
+ log.info(message, fields);
+ }
+ };
+
+ app.use(morgan(':method :url :status :response-time', { stream }));
+ }
+
+ app.use((() => {
+ // const router = express.Router();
+ // router.get('/wut', express.json(), (req, res, next) => {
+ // return res.status(500).send('Internal Error');
+ // });
+ // return router;
+
+ return eggspress('/wut', {
+ allowedMethods: ['GET'],
+ }, async (req, res, next) => {
+ // throw new Error('throwy error');
+ return res.status(200).send('test endpoint');
+ });
+ })());
+
+ (() => {
+ const onFinished = require('on-finished');
+ app.use((req, res, next) => {
+ onFinished(res, () => {
+ if ( res.statusCode !== 500 ) return;
+ if ( req.__error_handled ) return;
+ const alarm = services.get('alarm');
+ alarm.create('responded-500', 'server sent a 500 response', {
+ error: req.__error_source,
+ url: req.url,
+ method: req.method,
+ body: req.body,
+ headers: req.headers,
+ });
+ });
+ next();
+ });
+ })();
+
+ app.use(async function(req, res, next) {
+ // Express does not document that this can be undefined.
+ // The browser likely doesn't follow the HTTP/1.1 spec
+ // (bot client?) and express is handling this badly by
+ // not setting the header at all. (that's my theory)
+ if( req.hostname === undefined ) {
+ res.status(400).send(
+ 'Please verify your browser is up-to-date.'
+ );
+ return;
+ }
+
+ return next();
+ });
+
+ app.use(express.json({limit: '50mb'}));
+
+ const cookieParser = require('cookie-parser');
+ app.use(cookieParser({limit: '50mb'}));
+
+ // gzip compression for all requests
+ const compression = require('compression');
+ app.use(compression());
+
+ // Helmet and other security
+ const helmet = require('helmet');
+ app.use(helmet.noSniff());
+ app.use(helmet.hsts());
+ app.use(helmet.ieNoOpen());
+ app.use(helmet.permittedCrossDomainPolicies());
+ app.use(helmet.xssFilter());
+ // app.use(helmet.referrerPolicy());
+ app.disable('x-powered-by');
+
+ app.use(function (req, res, next) {
+ const origin = req.headers.origin;
+
+ if ( req.path === '/signup' || req.path === '/login' ) {
+ res.setHeader('Access-Control-Allow-Origin', origin ?? '*');
+ }
+ // Website(s) to allow to connect
+ if ( req.subdomains[0] === 'api' ) {
+ res.setHeader('Access-Control-Allow-Origin', origin ?? '*');
+ res.setHeader('Access-Control-Allow-Credentials', 'true');
+ }
+
+ // Request methods to allow
+ res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE');
+
+ const allowed_headers = [
+ "Origin", "X-Requested-With", "Content-Type", "Accept", "Authorization",
+ ];
+
+ // Request headers to allow
+ res.header("Access-Control-Allow-Headers", allowed_headers.join(', '));
+
+ // Set to true if you need the website to include cookies in the requests sent
+ // to the API (e.g. in case you use sessions)
+ // res.setHeader('Access-Control-Allow-Credentials', true);
+
+ //needed for SharedArrayBuffer
+ // res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
+ // res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp')
+ res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin');
+ // Pass to next layer of middleware
+ next();
+ });
+
+ // Options for all requests (for CORS)
+ app.options('/*', (_, res) => {
+ return res.sendStatus(200);
+ });
+ }
+
+ _register_commands (commands) {
+ commands.registerCommands('web', [
+ {
+ id: 'dismiss',
+ description: 'Dismiss the startup message',
+ handler: async () => {
+ const svc_devConsole = this.services.get('dev-console');
+ svc_devConsole.remove_widget(this.startup_widget);
+ }
+ }
+ ]);
+ }
+
+ print_puter_logo_() {
+ if ( this.global_config.env !== 'dev' ) return;
+ const logos = require('../fun/logos.js');
+ let last_logo = undefined;
+ for ( const logo of logos ) {
+ console.log('comparing', process.stdout.columns, logo.sz)
+ if ( logo.sz <= (process.stdout.columns ?? 0) ) {
+ last_logo = logo;
+ } else break;
+ }
+ if ( last_logo ) {
+ console.log('\x1B[34;1m' + last_logo.txt + '\x1B[0m');
+ }
+ }
+}
+
+module.exports = WebServerService;
diff --git a/packages/backend/src/services/abuse-prevention/AuthAuditService.js b/packages/backend/src/services/abuse-prevention/AuthAuditService.js
new file mode 100644
index 00000000..986a5345
--- /dev/null
+++ b/packages/backend/src/services/abuse-prevention/AuthAuditService.js
@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("../BaseService");
+const { DB_WRITE } = require("../database/consts");
+
+class AuthAuditService extends BaseService {
+ static MODULES = {
+ uuidv4: require('uuid').v4,
+ }
+ async _init () {
+ this.db = this.services.get('database').get(DB_WRITE, 'auth:audit');
+ }
+
+ async record (parameters) {
+ try {
+ await this._record(parameters);
+ } catch (err) {
+ this.errors.report('auth-audit-service.record', {
+ source: err,
+ trace: true,
+ alarm: true,
+ });
+ }
+ }
+
+ async _record ({ requester, action, body, extra }) {
+ const uid = 'aas-' + this.modules.uuidv4();
+
+ const json_values = {
+ requester: requester.serialize(),
+ body: body,
+ extra: extra ?? {},
+ };
+
+ let has_parse_error = 0;
+
+ for ( const k in json_values ) {
+ let value = json_values[k];
+ try {
+ value = JSON.stringify(value);
+ } catch (err) {
+ has_parse_error = 1;
+ value = { parse_error: err.message };
+ }
+ json_values[k] = value;
+ }
+
+ await this.db.write(
+ `INSERT INTO auth_audit (` +
+ `uid, ip_address, ua_string, action, ` +
+ `requester, body, extra, ` +
+ `has_parse_error` +
+ `) VALUES ( ?, ?, ?, ?, ?, ?, ?, ? )`,
+ [
+ uid,
+ requester.ip,
+ requester.ua,
+ action,
+ JSON.stringify(requester.serialize()),
+ JSON.stringify(body),
+ JSON.stringify(extra ?? {}),
+ has_parse_error,
+ ]
+ );
+ }
+}
+
+module.exports = {
+ AuthAuditService
+};
diff --git a/packages/backend/src/services/abuse-prevention/IdentificationService.js b/packages/backend/src/services/abuse-prevention/IdentificationService.js
new file mode 100644
index 00000000..0a8c5885
--- /dev/null
+++ b/packages/backend/src/services/abuse-prevention/IdentificationService.js
@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const BaseService = require("../BaseService");
+const { Context } = require("../../util/context");
+const config = require("../../config");
+
+class Requester {
+ constructor (o) {
+ for ( const k in o ) this[k] = o[k];
+ }
+ static create (o) {
+ return new Requester(o);
+ }
+ static from_request (req) {
+
+ const has_referer = req.headers['referer'] !== undefined;
+ let referer_url;
+ let referer_origin;
+ if ( has_referer ) {
+ try {
+ referer_url = new URL(req.headers['referer']);
+ referer_origin = referer_url.origin;
+ } catch (e) {
+ // URL is invalid; referer_url and referer_origin will be undefined
+ }
+ }
+
+ return new Requester({
+ ua: req.headers['user-agent'],
+ ip: req.connection.remoteAddress,
+ ip_forwarded: req.headers['x-forwarded-for'],
+ origin: req.headers['origin'],
+ referer: req.headers['referer'],
+ referer_origin,
+ });
+ }
+
+ is_puter_referer () {
+ const puter_origins = [
+ config.origin,
+ config.api_base_url,
+ ]
+ return puter_origins.includes(this.referer_origin);
+ }
+
+ is_puter_origin () {
+ const puter_origins = [
+ config.origin,
+ config.api_base_url,
+ ]
+ return puter_origins.includes(this.origin);
+ }
+
+ serialize () {
+ return {
+ ua: this.ua,
+ ip: this.ip,
+ ip_forwarded: this.ip_forwarded,
+ referer: this.referer,
+ referer_origin: this.referer_origin,
+ };
+ }
+
+}
+
+// DRY: (3/3) - src/util/context.js; move install() to base class
+class RequesterIdentificationExpressMiddleware extends AdvancedBase {
+ static MODULES = {
+ isbot: require('isbot'),
+ }
+ register_initializer (initializer) {
+ this.value_initializers_.push(initializer);
+ }
+ install (app) {
+ app.use(this.run.bind(this));
+ }
+ async run (req, res, next) {
+ const x = Context.get();
+
+ const requester = Requester.from_request(req);
+ const is_bot = this.modules.isbot(requester.ua);
+ requester.is_bot = is_bot;
+
+ x.set('requester', requester);
+ req.requester = requester;
+
+ if ( requester.is_bot ) {
+ this.log.info('bot detected', requester.serialize());
+ }
+
+ next();
+ }
+}
+
+class IdentificationService extends BaseService {
+ _construct () {
+ this.mw = new RequesterIdentificationExpressMiddleware();
+ }
+ _init () {
+ this.mw.log = this.log;
+ }
+ async ['__on_install.middlewares.context-aware'] (_, { app }) {
+ this.mw.install(app);
+ }
+}
+
+module.exports = {
+ IdentificationService,
+};
diff --git a/packages/backend/src/services/auth/ACLService.js b/packages/backend/src/services/auth/ACLService.js
new file mode 100644
index 00000000..a39c52d5
--- /dev/null
+++ b/packages/backend/src/services/auth/ACLService.js
@@ -0,0 +1,160 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../api/APIError");
+const { NodePathSelector } = require("../../filesystem/node/selectors");
+const { get_user } = require("../../helpers");
+const { Context } = require("../../util/context");
+const BaseService = require("../BaseService");
+const { AppUnderUserActorType, UserActorType, Actor, SystemActorType, AccessTokenActorType } = require("./Actor");
+
+class ACLService extends BaseService {
+ async check (actor, resource, mode) {
+ const result = await this._check_fsNode(actor, resource, mode);
+ if ( this.verbose ) console.log('LOGGING ACL CHECK', {
+ actor, mode,
+ // trace: (new Error()).stack,
+ result,
+ });
+ return result;
+ }
+
+ async _check_fsNode (actor, fsNode, mode) {
+ const context = Context.get();
+
+ actor = Actor.adapt(actor);
+
+ if ( actor.type instanceof SystemActorType ) {
+ return true;
+ }
+
+ const path_selector = fsNode.get_selector_of_type(NodePathSelector);
+ if ( path_selector && path_selector.value === '/') {
+ if (['list','see','read'].includes(mode)) {
+ return true;
+ }
+ return false;
+ }
+
+ // Access tokens only work if the authorizer has permission
+ if ( actor.type instanceof AccessTokenActorType ) {
+ const authorizer = actor.type.authorizer;
+ const authorizer_perm = await this._check_fsNode(authorizer, fsNode, mode);
+
+ if ( ! authorizer_perm ) return false;
+ }
+
+ // Hard rule: if app-under-user is accessing appdata directory, allow
+ if ( actor.type instanceof AppUnderUserActorType ) {
+ const appdata_path = `/${actor.type.user.username}/AppData/${actor.type.app.uid}`;
+ const svc_fs = await context.get('services').get('filesystem');
+ const appdata_node = await svc_fs.node(new NodePathSelector(appdata_path));
+
+ if (
+ await appdata_node.exists() && (
+ await appdata_node.is(fsNode) ||
+ await appdata_node.is_above(fsNode)
+ )
+ ) {
+ console.log('TRUE BECAUSE APPDATA')
+ return true;
+ }
+ }
+
+ // Hard rule: if actor is owner, allow
+ if ( actor.type instanceof UserActorType ) {
+ const owner = await fsNode.get('user_id');
+ if ( this.verbose ) {
+ const user = await get_user({ id: owner });
+ this.log.info(
+ `user ${user.username} is ` +
+ (owner == actor.type.user.id ? '' : 'not ') +
+ 'owner of ' + await fsNode.get('path'), {
+ actor_user_id: actor.type.user.id,
+ fsnode_user_id: owner,
+ }
+ );
+ }
+ if ( owner == actor.type.user.id ) {
+ return true;
+ }
+ }
+
+ // For these actors, deny if the user component is not the owner
+ // -> user
+ // -> app-under-user
+ if ( ! (actor.type instanceof AccessTokenActorType) ) {
+ const owner = await fsNode.get('user_id');
+ if ( owner != actor.type.user.id ) {
+ return false;
+ }
+ }
+
+ // app-under-user only works if the user also has permission
+ if ( actor.type instanceof AppUnderUserActorType ) {
+ const user_actor = new Actor({
+ type: new UserActorType({ user: actor.type.user }),
+ });
+ const user_perm = await this._check_fsNode(user_actor, fsNode, mode);
+
+ if ( ! user_perm ) return false;
+ }
+
+ const svc_permission = await context.get('services').get('permission');
+
+ const modes = this._higher_modes(mode);
+ let perm_fsNode = fsNode;
+ while ( ! await perm_fsNode.get('is-root') ) {
+ for ( const mode of modes ) {
+ const perm = await svc_permission.check(
+ actor,
+ `fs:${await perm_fsNode.get('uid')}:${mode}`
+ );
+ if ( perm ) {
+ console.log('TRUE BECAUSE PERMISSION', perm)
+ console.log(`fs:${await perm_fsNode.get('uid')}:${mode}`)
+ return true;
+ }
+ }
+ perm_fsNode = await perm_fsNode.getParent();
+ }
+
+ return false;
+ }
+
+ async get_safe_acl_error (actor, resource, mode) {
+ const can_see = await this.check(actor, resource, 'see');
+ if ( ! can_see ) {
+ return APIError.create('subject_does_not_exist');
+ }
+
+ return APIError.create('forbidden');
+ }
+
+ _higher_modes (mode) {
+ // If you want to X, you can do so with any of [...Y]
+ if ( mode === 'see' ) return ['see', 'list', 'read', 'write'];
+ if ( mode === 'list' ) return ['list', 'read', 'write'];
+ if ( mode === 'read' ) return ['read', 'write'];
+ if ( mode === 'write' ) return ['write'];
+ }
+}
+
+module.exports = {
+ ACLService,
+};
diff --git a/packages/backend/src/services/auth/Actor.js b/packages/backend/src/services/auth/Actor.js
new file mode 100644
index 00000000..2b128cfd
--- /dev/null
+++ b/packages/backend/src/services/auth/Actor.js
@@ -0,0 +1,204 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { Context } = require("../../util/context");
+const { get_user, get_app } = require("../../helpers");
+
+const PRIVATE_UID_NAMESPACE = '1757dc3f-8f04-4d77-b939-ff899045696d';
+const PRIVATE_UID_SECRET = 'bf03f0e52f5d93c83822ad8558c625277ce3dddff8dc4a5cb0d3c8493571f770';
+
+class Actor extends AdvancedBase {
+ static MODULES = {
+ uuidv5: require('uuid').v5,
+ crypto: require('crypto'),
+ }
+
+ static system_actor_ = null;
+ static get_system_actor () {
+ if ( ! this.system_actor_ ) {
+ this.system_actor_ = new Actor({
+ type: new SystemActorType(),
+ });
+ }
+ return this.system_actor_;
+ }
+
+ static async create (type, params) {
+ params = { ...params };
+ if ( params.user_uid ) {
+ params.user = await get_user({ uuid: params.user_uid });
+ }
+ if ( params.app_uid ) {
+ params.app = await get_app({ uid: params.app_uid });
+ }
+ return new Actor({
+ type: new type(params),
+ });
+ }
+
+ constructor (o, ...a) {
+ super(o, ...a);
+ for ( const k in o ) {
+ this[k] = o[k];
+ }
+ }
+ get uid () {
+ return this.type.uid;
+ }
+
+ /**
+ * Generate a cryptographically-secure deterministic UUID
+ * from an actor's UID.
+ */
+ get private_uid () {
+ // Pass the UUID through SHA-2 first because UUIDv5
+ // is not cryptographically secure (it uses SHA-1)
+ const hmac = this.modules.crypto.createHmac('sha256', PRIVATE_UID_SECRET)
+ .update(this.uid)
+ .digest('hex');
+
+ // Generate a UUIDv5 from the HMAC
+ // Note: this effectively does an additional SHA-1 hash,
+ // but this is done only to format the result as a UUID
+ // and not for cryptographic purposes
+ let str = this.modules.uuidv5(hmac, PRIVATE_UID_NAMESPACE);
+
+ // Uppercase UUID to avoid inference of what uuid library is being used
+ str = ('' + str).toUpperCase();
+ return str;
+ }
+
+ clone () {
+ return new Actor({
+ type: this.type,
+ });
+ }
+
+ get_related_actor (type_class) {
+ const actor = this.clone();
+ actor.type = this.type.get_related_type(type_class);
+ return actor;
+ }
+}
+
+class SystemActorType {
+ constructor (o, ...a) {
+ // super(o, ...a);
+ for ( const k in o ) {
+ this[k] = o[k];
+ }
+ }
+ get uid () {
+ return 'system';
+ }
+ get_related_type (type_class) {
+ if ( type_class === SystemActorType ) {
+ return this;
+ }
+ throw new Error(`cannot get ${type_class.name} from ${this.constructor.name}`)
+ }
+}
+
+class UserActorType {
+ constructor (o, ...a) {
+ // super(o, ...a);
+ for ( const k in o ) {
+ this[k] = o[k];
+ }
+ }
+ get uid () {
+ return 'user:' + this.user.uuid;
+ }
+ get_related_type (type_class) {
+ if ( type_class === UserActorType ) {
+ return this;
+ }
+ throw new Error(`cannot get ${type_class.name} from ${this.constructor.name}`)
+ }
+}
+class AppUnderUserActorType {
+ constructor (o, ...a) {
+ // super(o, ...a);
+ for ( const k in o ) {
+ this[k] = o[k];
+ }
+ }
+ get uid () {
+ return 'app-under-user:' + this.user.uuid + ':' + this.app.uid;
+ }
+ get_related_type (type_class) {
+ if ( type_class === UserActorType ) {
+ return new UserActorType({ user: this.user });
+ }
+ if ( type_class === AppUnderUserActorType ) {
+ return this;
+ }
+ throw new Error(`cannot get ${type_class.name} from ${this.constructor.name}`)
+ }
+}
+
+class AccessTokenActorType {
+ // authorizer: an Actor who authorized the token
+ // authorized: an Actor who is authorized by the token
+ // token: a string
+ constructor (o, ...a) {
+ // super(o, ...a);
+ for ( const k in o ) {
+ this[k] = o[k];
+ }
+ }
+ get uid () {
+ return 'access-token:' + this.authorizer.uid +
+ ':' + ( this.authorized?.uid ?? '' ) +
+ ':' + this.token;
+ }
+ get_related_actor () {
+ // This would be dangerous because of ambiguity
+ // between authorizer and authorized
+ throw new Error('cannot call get_related_actor on ' + this.constructor.name);
+ }
+}
+
+Actor.adapt = function (actor) {
+ actor = actor || Context.get('actor');
+
+ if ( actor?.username ) {
+ const user = actor;
+ actor = new Actor({
+ type: new UserActorType({ user }),
+ });
+ }
+ // Legacy: if actor is undefined, use the user in the context
+ if ( ! actor ) {
+ const user = Context.get('user');
+ actor = new Actor({
+ type: new UserActorType({ user }),
+ });
+ }
+
+ return actor;
+}
+
+module.exports = {
+ Actor,
+ SystemActorType,
+ UserActorType,
+ AppUnderUserActorType,
+ AccessTokenActorType,
+}
diff --git a/packages/backend/src/services/auth/AuthService.js b/packages/backend/src/services/auth/AuthService.js
new file mode 100644
index 00000000..29218622
--- /dev/null
+++ b/packages/backend/src/services/auth/AuthService.js
@@ -0,0 +1,267 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Actor, UserActorType, AppUnderUserActorType, AccessTokenActorType } = require("./Actor");
+const BaseService = require("../BaseService");
+const { get_user, get_app } = require("../../helpers");
+const { Context } = require("../../util/context");
+const APIError = require("../../api/APIError");
+const { DB_WRITE } = require("../database/consts");
+
+const APP_ORIGIN_UUID_NAMESPACE = '33de3768-8ee0-43e9-9e73-db192b97a5d8';
+
+class AuthService extends BaseService {
+ static MODULES = {
+ jwt: require('jsonwebtoken'),
+ uuidv5: require('uuid').v5,
+ uuidv4: require('uuid').v4,
+ }
+
+ async _init () {
+ this.db = await this.services.get('database').get(DB_WRITE, 'auth');
+ }
+
+ async authenticate_from_token (token) {
+ const decoded = this.modules.jwt.verify(
+ token,
+ this.global_config.jwt_secret
+ );
+
+ if ( ! decoded.hasOwnProperty('type') ) {
+ const user = await this.db.requireRead(
+ "SELECT * FROM `user` WHERE `uuid` = ? LIMIT 1",
+ [decoded.uuid],
+ );
+
+ if ( ! user[0] ) {
+ throw APIError.create('token_auth_failed');
+ }
+
+ if ( user[0].suspended ) {
+ throw APIError.create('account_suspended');
+ }
+
+ const actor_type = new UserActorType({
+ user: user[0],
+ });
+
+ return new Actor({
+ user_uid: decoded.uuid,
+ type: actor_type,
+ });
+ }
+
+ if ( decoded.type === 'app-under-user' ) {
+ const user = await get_user({ uuid: decoded.user_uid });
+ if ( ! user ) {
+ throw APIError.create('token_auth_failed');
+ }
+
+ const app = await get_app({ uid: decoded.app_uid });
+ if ( ! app ) {
+ throw APIError.create('token_auth_failed');
+ }
+
+ const actor_type = new AppUnderUserActorType({
+ user,
+ app,
+ });
+
+ return new Actor({
+ user_uid: decoded.user_uid,
+ app_uid: decoded.app_uid,
+ type: actor_type,
+ });
+ }
+
+ if ( decoded.type === 'access-token' ) {
+ const token = decoded.token_uid;
+ console.log('DECODED', decoded);
+ if ( ! token ) {
+ throw APIError.create('token_auth_failed');
+ }
+
+ const user_uid = decoded.user_uid;
+ if ( ! user_uid ) {
+ throw APIError.create('token_auth_failed');
+ }
+
+ const app_uid = decoded.app_uid;
+
+ const authorizer = ( user_uid && app_uid )
+ ? await Actor.create(AppUnderUserActorType, { user_uid, app_uid })
+ : await Actor.create(UserActorType, { user_uid });
+
+ const authorized = Context.get('actor');
+
+ const actor_type = new AccessTokenActorType({
+ token, authorizer, authorized,
+ });
+
+ return new Actor({
+ user_uid,
+ app_uid,
+ type: actor_type,
+ });
+ }
+
+ throw APIError.create('token_auth_failed');
+ }
+
+ get_user_app_token (app_uid) {
+ const actor = Context.get('actor');
+ const actor_type = actor.type;
+
+ if ( ! (actor_type instanceof UserActorType) ) {
+ throw APIError.create('forbidden');
+ }
+
+ this.log.info(`generating user-app token for app ${app_uid} and user ${actor_type.user.uuid}`, {
+ app_uid,
+ user_uid: actor_type.user.uuid,
+ })
+
+ const token = this.modules.jwt.sign(
+ {
+ type: 'app-under-user',
+ version: '0.0.0',
+ user_uid: actor_type.user.uuid,
+ app_uid,
+ },
+ this.global_config.jwt_secret,
+ );
+
+ return token;
+ }
+
+ async create_access_token (authorizer, permissions) {
+ const jwt_obj = {};
+ const authorizer_obj = {};
+ if ( authorizer.type instanceof UserActorType ) {
+ Object.assign(authorizer_obj, {
+ authorizer_user_id: authorizer.type.user.id,
+ });
+ const user = await get_user({ id: authorizer.type.user.id });
+ jwt_obj.user_uid = user.uuid;
+ }
+ else if ( authorizer.type instanceof AppUnderUserActorType ) {
+ Object.assign(authorizer_obj, {
+ authorizer_user_id: authorizer.type.user.id,
+ authorizer_app_id: authorizer.type.app.id,
+ });
+ const user = await get_user({ id: authorizer.type.user.id });
+ jwt_obj.user_uid = user.uuid;
+ const app = await get_app({ id: authorizer.type.app.id });
+ jwt_obj.app_uid = app.uid;
+ }
+ else {
+ throw APIError.create('forbidden');
+ }
+
+ const uuid = this.modules.uuidv4();
+
+ const jwt = this.modules.jwt.sign({
+ type: 'access-token',
+ version: '0.0.0',
+ token_uid: uuid,
+ ...jwt_obj,
+ }, this.global_config.jwt_secret);
+
+ for ( const permmission_spec of permissions ) {
+ let [permission, extra] = permmission_spec;
+
+ const svc_permission = await Context.get('services').get('permission');
+ permission = await svc_permission._rewrite_permission(permission);
+
+ const insert_object = {
+ token_uid: uuid,
+ ...authorizer_obj,
+ permission,
+ extra: JSON.stringify(extra ?? {}),
+ };
+ const cols = Object.keys(insert_object).join(', ');
+ const vals = Object.values(insert_object).map(v => '?').join(', ');
+ await this.db.write(
+ 'INSERT INTO `access_token_permissions` ' +
+ `(${cols}) VALUES (${vals})`,
+ Object.values(insert_object),
+ );
+ }
+
+ return jwt;
+ }
+
+ async get_user_app_token_from_origin (origin) {
+ origin = this._origin_from_url(origin);
+ const app_uid = await this._app_uid_from_origin(origin);
+
+ // Determine if the app exists
+ const apps = await this.db.read(
+ "SELECT * FROM `apps` WHERE `uid` = ? LIMIT 1",
+ [app_uid],
+ );
+
+ if ( apps[0] ) {
+ return this.get_user_app_token(app_uid);
+ }
+
+ this.log.info(`creating app ${app_uid} from origin ${origin}`);
+
+ const name = app_uid;
+ const title = app_uid;
+ const description = `App created from origin ${origin}`;
+ const index_url = origin;
+ const owner_user_id = null;
+
+ // Create the app
+ await this.db.write(
+ 'INSERT INTO `apps` ' +
+ '(`uid`, `name`, `title`, `description`, `index_url`, `owner_user_id`) ' +
+ 'VALUES (?, ?, ?, ?, ?, ?)',
+ [app_uid, name, title, description, index_url, owner_user_id],
+ );
+
+ return this.get_user_app_token(app_uid);
+ }
+
+ async app_uid_from_origin (origin) {
+ origin = this._origin_from_url(origin);
+ return await this._app_uid_from_origin(origin);
+ }
+
+ async _app_uid_from_origin (origin) {
+ // UUIDV5
+ const uuid = this.modules.uuidv5(origin, APP_ORIGIN_UUID_NAMESPACE);
+ return `app-${uuid}`;
+ }
+
+ _origin_from_url ( url ) {
+ try {
+ const parsedUrl = new URL(url);
+ // Origin is protocol + hostname + port
+ return `${parsedUrl.protocol}//${parsedUrl.hostname}${parsedUrl.port ? `:${parsedUrl.port}` : ''}`;
+ } catch (error) {
+ console.error('Invalid URL:', error.message);
+ return null;
+ }
+ }
+}
+
+module.exports = {
+ AuthService,
+};
diff --git a/packages/backend/src/services/auth/PermissionService.js b/packages/backend/src/services/auth/PermissionService.js
new file mode 100644
index 00000000..fa32c502
--- /dev/null
+++ b/packages/backend/src/services/auth/PermissionService.js
@@ -0,0 +1,430 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { get_user, get_app } = require("../../helpers");
+const BaseService = require("../BaseService");
+const { DB_WRITE } = require("../database/consts");
+const { UserActorType, Actor, AppUnderUserActorType, AccessTokenActorType } = require("./Actor");
+
+const default_implicit_user_app_permissions = {
+ 'driver:helloworld:greet': {},
+ 'driver:puter-kvstore': {},
+ 'driver:puter-ocr:recognize': {},
+ 'driver:puter-chat-completion': {},
+ 'driver:puter-image-generation': {},
+ 'driver:puter-tts': {},
+ 'driver:puter-apps': {},
+ 'driver:puter-subdomains': {},
+};
+
+const implicit_user_app_permissions = [
+ {
+ id: 'builtin-apps',
+ apps: [
+ 'app-0bef044f-918f-4cbf-a0c0-b4a17ee81085', // about
+ 'app-838dfbc4-bf8b-48c2-b47b-c4adc77fab58', // editor
+ 'app-58282b08-990a-4906-95f7-fa37ff92452b', // draw
+ 'app-0087b701-da09-4f49-a37d-2d6bcabc81ee', // minipaint
+ 'app-3fea7529-266e-47d9-8776-31649cd06557', // terminal
+ 'app-5584fbf7-ed69-41fc-99cd-85da21b1ef51', // camera
+ 'app-7bdca1a4-6373-4c98-ad97-03ff2d608ca1', // recorder
+ 'app-240a43f4-43b1-49bc-b9fc-c8ae719dab77', // dev-center
+ 'app-a2ae72a4-1ba3-4a29-b5c0-6de1be5cf178', // app-center
+ 'app-74378e84-b9cd-5910-bcb1-3c50fa96d6e7', // https://nj.puter.site
+ 'app-13a38aeb-f9f6-54f0-9bd3-9d4dd655ccfe', // https://cdpn.io
+ 'app-dce8f797-82b0-5d95-a2f8-ebe4d71b9c54', // https://null.jsbin.com
+ 'app-93005ce0-80d1-50d9-9b1e-9c453c375d56', // https://markus.puter.com
+ ],
+ permissions: {
+ 'driver:helloworld:greet': {},
+ 'driver:puter-ocr:recognize': {},
+ 'driver:puter-kvstore:get': {},
+ 'driver:puter-kvstore:set': {},
+ 'driver:puter-kvstore:del': {},
+ 'driver:puter-kvstore:list': {},
+ 'driver:puter-kvstore:flush': {},
+ 'driver:puter-chat-completion:complete': {},
+ 'driver:puter-image-generation:generate': {},
+ 'driver:puter-analytics:create_trace': {},
+ 'driver:puter-analytics:record': {},
+ },
+ },
+ {
+ id: 'local-testing',
+ apps: [
+ 'app-a392f3e5-35ca-5dac-ae10-785696cc7dec', // https://localhost
+ 'app-a6263561-6a84-5d52-9891-02956f9fac65', // https://127.0.0.1
+ 'app-26149f0b-8304-5228-b995-772dadcf410e', // http://localhost
+ 'app-c2e27728-66d9-54dd-87cd-6f4e9b92e3e3', // http://127.0.0.1
+ ],
+ permissions: {
+ 'driver:helloworld:greet': {},
+ 'driver:puter-ocr:recognize': {},
+ 'driver:puter-kvstore:get': {},
+ 'driver:puter-kvstore:set': {},
+ 'driver:puter-kvstore:del': {},
+ 'driver:puter-kvstore:list': {},
+ 'driver:puter-kvstore:flush': {},
+ },
+ },
+];
+
+class PermissionRewriter {
+ static create ({ id, matcher, rewriter }) {
+ return new PermissionRewriter({ id, matcher, rewriter });
+ }
+
+ constructor ({ id, matcher, rewriter }) {
+ this.id = id;
+ this.matcher = matcher;
+ this.rewriter = rewriter;
+ }
+
+ matches (permission) {
+ return this.matcher(permission);
+ }
+
+ async rewrite (permission) {
+ return await this.rewriter(permission);
+ }
+}
+
+class PermissionUtil {
+ static unescape_permission_component (component) {
+ let unescaped_str = '';
+ const STATE_NORMAL = {};
+ const STATE_ESCAPE = {};
+ let state = STATE_NORMAL;
+ const const_escapes = { C: ':' };
+ for ( let i = 0 ; i < component.length ; i++ ) {
+ const c = component[i];
+ if ( state === STATE_NORMAL ) {
+ if ( c === '\\' ) {
+ state = STATE_ESCAPE;
+ } else {
+ unescaped_str += c;
+ }
+ } else if ( state === STATE_ESCAPE ) {
+ unescaped_str += const_escapes.hasOwnProperty(c)
+ ? const_escapes[c] : c;
+ state = STATE_NORMAL;
+ }
+ }
+ return unescaped_str;
+ }
+
+ static split (permission) {
+ return permission
+ .split(':')
+ .map(PermissionUtil.unescape_permission_component)
+ ;
+ }
+}
+
+class PermissionService extends BaseService {
+ async _init () {
+ this.db = this.services.get('database').get(DB_WRITE, 'permissions');
+ this._register_commands(this.services.get('commands'));
+
+ this._permission_rewriters = [];
+ }
+
+ async _rewrite_permission (permission) {
+ for ( const rewriter of this._permission_rewriters ) {
+ if ( ! rewriter.matches(permission) ) continue;
+ permission = await rewriter.rewrite(permission);
+ }
+ return permission;
+ }
+
+
+ async check (actor, permission) {
+ permission = await this._rewrite_permission(permission);
+
+ this.log.info(`checking permission ${permission} for actor ${actor.uid}`, {
+ actor: actor.uid,
+ permission,
+ });
+ // For now we're only checking driver permissions, and users have all of them
+ if ( actor.type instanceof UserActorType ) {
+ return {};
+ }
+
+ if ( actor.type instanceof AccessTokenActorType ) {
+ return await this.check_access_token_permission(
+ actor.type.authorizer, actor.type.token, permission
+ );
+ }
+
+ // Prevent undefined behaviour
+ if ( ! (actor.type instanceof AppUnderUserActorType) ) {
+ throw new Error('actor must be an app under a user');
+ }
+
+ // Now it's an app under a user
+ const app_uid = actor.type.app.uid;
+ return await this.check_user_app_permission(actor, app_uid, permission);
+ }
+
+ async check_access_token_permission (authorizer, token, permission) {
+ // Authorizer must have permission
+ const authorizer_permission = await this.check(authorizer, permission);
+ if ( ! authorizer_permission ) return false;
+
+ const rows = await this.db.read(
+ 'SELECT * FROM `access_token_permissions` ' +
+ 'WHERE `token_uid` = ? AND `permission` = ?',
+ [
+ token,
+ permission,
+ ]
+ );
+
+ // Token must have permission
+ if ( ! rows[0] ) return undefined;
+
+ return rows[0].extra;
+ }
+
+ async check_user_app_permission (actor, app_uid, permission) {
+ permission = await this._rewrite_permission(permission);
+
+ let app = await get_app({ uid: app_uid });
+ if ( ! app ) app = await get_app({ name: app_uid });
+ const app_id = app.id;
+
+ const parent_perms = [];
+ {
+ // We don't use PermissionUtil.split here because it unescapes
+ // components; we want to keep the components escaped for matching.
+ const parts = permission.split(':');
+
+ // Add sub-permissions
+ for ( let i = 1 ; i < parts.length ; i++ ) {
+ parent_perms.push(parts.slice(0, i + 1).join(':'));
+ }
+ }
+ parent_perms.reverse();
+
+ for ( const permission of parent_perms ) {
+ // Check hardcoded permissions
+ if ( default_implicit_user_app_permissions[permission] ) {
+ return default_implicit_user_app_permissions[permission];
+ }
+
+ // Check implicit permissions
+ const implicit_permissions = {};
+ for ( const implicit_permission of implicit_user_app_permissions ) {
+ if ( implicit_permission.apps.includes(app_uid) ) {
+ implicit_permissions[permission] = implicit_permission.permissions[permission];
+ }
+ }
+ if ( implicit_permissions[permission] ) {
+ return implicit_permissions[permission];
+ }
+ }
+
+ // My biggest gripe with SQL is doing string manipulation for queries.
+ // If the grammar for SQL was simpler we could model it, write this as
+ // data, and even implement macros for common patterns.
+ let sql_perm = parent_perms.map((perm) =>
+ `\`permission\` = ?`).join(' OR ');
+ if ( parent_perms.length > 1 ) sql_perm = '(' + sql_perm + ')';
+
+ // SELECT permission
+ const rows = await this.db.read(
+ 'SELECT * FROM `user_to_app_permissions` ' +
+ 'WHERE `user_id` = ? AND `app_id` = ? AND ' +
+ sql_perm,
+ [
+ actor.type.user.id,
+ app_id,
+ ...parent_perms,
+ ]
+ );
+
+ if ( ! rows[0] ) return undefined;
+
+ return rows[0].extra;
+ }
+
+ async grant_user_app_permission (actor, app_uid, permission, extra = {}, meta) {
+ permission = await this._rewrite_permission(permission);
+
+ let app = await get_app({ uid: app_uid });
+ if ( ! app ) app = await get_app({ name: app_uid });
+
+ const app_id = app.id;
+
+ // UPSERT permission
+ await this.db.write(
+ 'INSERT INTO `user_to_app_permissions` (`user_id`, `app_id`, `permission`, `extra`) ' +
+ 'VALUES (?, ?, ?, ?) ' +
+ this.db.case({
+ mysql: 'ON DUPLICATE KEY UPDATE `extra` = ?',
+ otherwise: 'ON CONFLICT(`user_id`, `app_id`, `permission`) DO UPDATE SET `extra` = ?',
+ }),
+ [
+ actor.type.user.id,
+ app_id,
+ permission,
+ JSON.stringify(extra),
+ JSON.stringify(extra),
+ ]
+ );
+
+ // INSERT audit table
+ const audit_values = {
+ user_id: actor.type.user.id,
+ user_id_keep: actor.type.user.id,
+ app_id: app_id,
+ app_id_keep: app_id,
+ permission,
+ action: 'grant',
+ reason: meta?.reason || 'granted via PermissionService',
+ };
+
+ const sql_cols = Object.keys(audit_values).map((key) => `\`${key}\``).join(', ');
+ const sql_vals = Object.keys(audit_values).map((key) => `?`).join(', ');
+
+ await this.db.write(
+ 'INSERT INTO `audit_user_to_app_permissions` (' + sql_cols + ') ' +
+ 'VALUES (' + sql_vals + ')',
+ Object.values(audit_values)
+ );
+ }
+
+ async revoke_user_app_permission (actor, app_uid, permission, meta) {
+ permission = await this._rewrite_permission(permission);
+
+ // For now, actor MUST be a user
+ if ( ! (actor.type instanceof UserActorType) ) {
+ throw new Error('actor must be a user');
+ }
+
+ let app = await get_app({ uid: app_uid });
+ if ( ! app ) app = await get_app({ name: app_uid });
+ const app_id = app.id;
+
+ // DELETE permission
+ await this.db.write(
+ 'DELETE FROM `user_to_app_permissions` ' +
+ 'WHERE `user_id` = ? AND `app_id` = ? AND `permission` = ?',
+ [
+ actor.type.user.id,
+ app_id,
+ permission,
+ ]
+ );
+
+ // INSERT audit table
+ const audit_values = {
+ user_id: actor.type.user.id,
+ user_id_keep: actor.type.user.id,
+ app_id: app_id,
+ app_id_keep: app_id,
+ permission,
+ action: 'revoke',
+ reason: meta?.reason || 'revoked via PermissionService',
+ };
+
+ const sql_cols = Object.keys(audit_values).map((key) => `\`${key}\``).join(', ');
+ const sql_vals = Object.keys(audit_values).map((key) => `?`).join(', ');
+
+ await this.db.write(
+ 'INSERT INTO `audit_user_to_app_permissions` (' + sql_cols + ') ' +
+ 'VALUES (' + sql_vals + ')',
+ Object.values(audit_values)
+ );
+ }
+
+ async revoke_user_app_all (actor, app_uid, meta) {
+ // For now, actor MUST be a user
+ if ( ! (actor.type instanceof UserActorType) ) {
+ throw new Error('actor must be a user');
+ }
+
+ let app = await get_app({ uid: app_uid });
+ if ( ! app ) app = await get_app({ name: app_uid });
+ const app_id = app.id;
+
+ // DELETE permissions
+ await this.db.write(
+ 'DELETE FROM `user_to_app_permissions` ' +
+ 'WHERE `user_id` = ? AND `app_id` = ?',
+ [
+ actor.type.user.id,
+ app_id,
+ ]
+ );
+
+ // INSERT audit table
+ const audit_values = {
+ user_id: actor.type.user.id,
+ user_id_keep: actor.type.user.id,
+ app_id: app_id,
+ app_id_keep: app_id,
+ permission: '*',
+ action: 'revoke',
+ reason: meta?.reason || 'revoked all via PermissionService',
+ };
+
+ const sql_cols = Object.keys(audit_values).map((key) => `\`${key}\``).join(', ');
+ const sql_vals = Object.keys(audit_values).map((key) => `?`).join(', ');
+
+ await this.db.write(
+ 'INSERT INTO `audit_user_to_app_permissions` (' + sql_cols + ') ' +
+ 'VALUES (' + sql_vals + ')',
+ Object.values(audit_values)
+ );
+ }
+
+ register_rewriter (translator) {
+ if ( ! (translator instanceof PermissionRewriter) ) {
+ throw new Error('translator must be a PermissionRewriter');
+ }
+
+ this._permission_rewriters.push(translator);
+ }
+
+ _register_commands (commands) {
+ commands.registerCommands('perms', [
+ {
+ id: 'grant-user-app',
+ handler: async (args, log) => {
+ const [ username, app_uid, permission, extra ] = args;
+
+ // actor from username
+ const actor = new Actor({
+ type: new UserActorType({
+ user: await get_user({ username }),
+ }),
+ })
+
+ await this.grant_user_app_permission(actor, app_uid, permission, extra);
+ }
+ }
+ ]);
+ }
+}
+
+module.exports = {
+ PermissionRewriter,
+ PermissionUtil,
+ PermissionService,
+};
diff --git a/packages/backend/src/services/database/BaseDatabaseAccessService.js b/packages/backend/src/services/database/BaseDatabaseAccessService.js
new file mode 100644
index 00000000..7682bea5
--- /dev/null
+++ b/packages/backend/src/services/database/BaseDatabaseAccessService.js
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const BaseService = require("../BaseService");
+
+class BaseDatabaseAccessService extends BaseService {
+ case ( choices ) {
+ const engine_name = this.constructor.ENGINE_NAME;
+ if ( choices.hasOwnProperty(engine_name) ) {
+ return choices[engine_name];
+ }
+ return choices.otherwise;
+ }
+
+ // Call get() with an access mode and a scope.
+ // Right now it just returns `this`, but in the
+ // future it can be used to audit the behaviour
+ // of other services or handle service-specific
+ // database optimizations.
+ get () {
+ return this;
+ }
+
+ read (query, params) {
+ return this._read(query, params);
+ }
+
+ pread (query, params) {
+ return this._read(query, params, { use_primary: true });
+ }
+
+ write (query, params) {
+ return this._write(query, params);
+ }
+
+ batch_write (statements) {
+ return this._batch_write(statements);
+ }
+
+ /**
+ * requireRead will fallback to the primary database
+ * when a read-replica configuration is in use;
+ * otherwise it behaves the same as `read()`.
+ *
+ * @param {string} query
+ * @param {array} params
+ * @returns {Promise<*>}
+ */
+ requireRead (query, params) {
+ return this._requireRead(query, params);
+ }
+}
+
+module.exports = {
+ BaseDatabaseAccessService,
+};
diff --git a/packages/backend/src/services/database/SqliteDatabaseAccessService.js b/packages/backend/src/services/database/SqliteDatabaseAccessService.js
new file mode 100644
index 00000000..758805bd
--- /dev/null
+++ b/packages/backend/src/services/database/SqliteDatabaseAccessService.js
@@ -0,0 +1,157 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { BaseDatabaseAccessService } = require("./BaseDatabaseAccessService");
+
+class SqliteDatabaseAccessService extends BaseDatabaseAccessService {
+ static ENGINE_NAME = 'sqlite';
+
+ static MODULES = {
+ // Documentation calls it 'Database'; it's new-able so
+ // I'll stick with their convention over ours.
+ Database: require('better-sqlite3'),
+ };
+
+ async _init () {
+ const require = this.require;
+ const Database = require('better-sqlite3');
+
+ this._register_commands(this.services.get('commands'));
+
+ const fs = require('fs');
+ const path_ = require('path');
+ const do_setup = ! fs.existsSync(this.config.path);
+
+ this.db = new Database(this.config.path);
+
+ if ( do_setup ) {
+ const sql_files = [
+ '0001_create-tables.sql',
+ '0002_add-default-apps.sql',
+ ].map(p => path_.join(__dirname, 'sqlite_setup', p));
+ const fs = require('fs');
+ for ( const filename of sql_files ) {
+ const contents = fs.readFileSync(filename, 'utf8');
+ this.db.exec(contents);
+ }
+ }
+
+ // Create the tables if they don't exist.
+ const check =
+ `SELECT name FROM sqlite_master WHERE type='table' AND name='fsentries'`;
+ const rows = await this.db.prepare(check).all();
+ if ( rows.length === 0 ) {
+ throw new Error('it works');
+ }
+ }
+
+ async _read (query, params = []) {
+ query = this.sqlite_transform_query_(query);
+ params = this.sqlite_transform_params_(params);
+ return this.db.prepare(query).all(...params);
+ }
+
+ async _requireRead (query, params) {
+ return this._read(query, params);
+ }
+
+ async _write (query, params) {
+ query = this.sqlite_transform_query_(query);
+ params = this.sqlite_transform_params_(params);
+
+ try {
+ const stmt = this.db.prepare(query);
+ const info = stmt.run(...params);
+
+ return {
+ insertId: info.lastInsertRowid,
+ anyRowsAffected: info.changes > 0,
+ };
+ } catch ( e ) {
+ console.error(e);
+ console.log('everything', {
+ query, params,
+ })
+ console.log(params.map(p => typeof p));
+ // throw e;
+ }
+ }
+
+ async _batch_write (entries) {
+ this.db.transaction(() => {
+ for ( let { statement, values } of entries ) {
+ statement = this.sqlite_transform_query_(statement);
+ values = this.sqlite_transform_params_(values);
+ this.db.prepare(statement).run(values);
+ }
+ })();
+ }
+
+
+ sqlite_transform_query_ (query) {
+ // replace `now()` with `datetime('now')`
+ query = query.replace(/now\(\)/g, 'datetime(\'now\')');
+
+ return query;
+ }
+
+ sqlite_transform_params_ (params) {
+ return params.map(p => {
+ if ( typeof p === 'boolean' ) {
+ return p ? 1 : 0;
+ }
+ return p;
+ });
+ }
+
+ _register_commands (commands) {
+ commands.registerCommands('sqlite', [
+ {
+ id: 'execfile',
+ description: 'execute a file',
+ handler: async (args, log) => {
+ try {
+ const [filename] = args;
+ const fs = require('fs');
+ const contents = fs.readFileSync(filename, 'utf8');
+ this.db.exec(contents);
+ } catch (err) {
+ log.error(err.message);
+ }
+ }
+ },
+ {
+ id: 'read',
+ description: 'read a query',
+ handler: async (args, log) => {
+ try {
+ const [query] = args;
+ const rows = this._read(query, []);
+ log.log(rows);
+ } catch (err) {
+ log.error(err.message);
+ }
+ }
+ },
+ ])
+ }
+}
+
+module.exports = {
+ SqliteDatabaseAccessService,
+};
diff --git a/packages/backend/src/services/database/constructs.js b/packages/backend/src/services/database/constructs.js
new file mode 100644
index 00000000..f6ca8d4a
--- /dev/null
+++ b/packages/backend/src/services/database/constructs.js
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+/**
+ * Statement simply holds a string that represents a SQL statement
+ * and an array of parameters to be used with the statement.
+ *
+ * This is meant to be used via the database access service when
+ * performing batch operations.
+ */
+const Statement = function Statement ({ statement, values }) {
+ // For now we just return an identical object.
+ return {
+ statement, values,
+ };
+}
+
+module.exports = {
+ Statement,
+};
diff --git a/packages/backend/src/services/database/consts.js b/packages/backend/src/services/database/consts.js
new file mode 100644
index 00000000..191932b2
--- /dev/null
+++ b/packages/backend/src/services/database/consts.js
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+module.exports = {
+ // Douglas Crockford doesn't like Symbol
+ // https://youtu.be/XFTOG895C7c?t=1469
+ // I think for this use case his argument would
+ // be "just use { label: 'DB_READ' } instead"
+ // but I've been using object references like
+ // that for years and it's refreshing not to
+ // need to assign an arbitrary property name
+ // to my debugging label.
+ // This is a pretty long comment for such a small
+ // file but nothing else is going to go in this
+ // file so it might as well have a long comment
+ // in it because if somebody is reading this file
+ // they're probably looking to find some secret
+ // undocumented constants and there aren't any
+ // so this comment will hopefully counter-balance
+ // the disappointment from that.
+ DB_READ: Symbol('DB_READ'),
+ DB_WRITE: Symbol('DB_WRITE'),
+};
diff --git a/packages/backend/src/services/database/sqlite_setup/0001_create-tables.sql b/packages/backend/src/services/database/sqlite_setup/0001_create-tables.sql
new file mode 100644
index 00000000..cf06a984
--- /dev/null
+++ b/packages/backend/src/services/database/sqlite_setup/0001_create-tables.sql
@@ -0,0 +1,390 @@
+-- drop all tables
+
+DROP TABLE IF EXISTS `monthly_usage_counts`;
+DROP TABLE IF EXISTS `access_token_permissions`;
+DROP TABLE IF EXISTS `auth_audit`;
+DROP TABLE IF EXISTS `general_analytics`;
+DROP TABLE IF EXISTS `audit_user_to_app_permissions`;
+DROP TABLE IF EXISTS `user_to_app_permissions`;
+DROP TABLE IF EXISTS `service_usage_monthly`;
+DROP TABLE IF EXISTS `rl_usage_fixed_window`;
+DROP TABLE IF EXISTS `app_update_audit`;
+DROP TABLE IF EXISTS `user_update_audit`;
+DROP TABLE IF EXISTS `storage_audit`;
+DROP TABLE IF EXISTS `user`;
+DROP TABLE IF EXISTS `subdomains`;
+DROP TABLE IF EXISTS `kv`;
+DROP TABLE IF EXISTS `fsentry_versions`;
+DROP TABLE IF EXISTS `fsentries`;
+DROP TABLE IF EXISTS `feedback`;
+DROP TABLE IF EXISTS `app_opens`;
+DROP TABLE IF EXISTS `app_filetype_association`;
+DROP TABLE IF EXISTS `apps`;
+
+CREATE TABLE `apps` (
+ `id` INTEGER PRIMARY KEY,
+ `uid` char(40) NOT NULL UNIQUE,
+ `owner_user_id` int(10) DEFAULT NULL, -- changed by: 0011
+ `icon` longtext,
+ `name` varchar(100) NOT NULL UNIQUE,
+ `title` varchar(100) NOT NULL,
+ `description` text,
+ `godmode` tinyint(1) DEFAULT '0',
+ `maximize_on_start` tinyint(1) DEFAULT '0',
+ `index_url` text NOT NULL,
+ `approved_for_listing` tinyint(1) DEFAULT '0',
+ `approved_for_opening_items` tinyint(1) DEFAULT '0',
+ `approved_for_incentive_program` tinyint(1) DEFAULT '0',
+ `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ `last_review` timestamp NULL DEFAULT NULL,
+
+ -- 0006
+ `tags` VARCHAR(255),
+ -- 0015
+ `app_owner` int(10) DEFAULT NULL,
+ FOREIGN KEY (`app_owner`) REFERENCES `apps` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+CREATE TABLE `app_filetype_association` (
+ `id` INTEGER PRIMARY KEY,
+ `app_id` int(10) NOT NULL,
+ `type` varchar(60) NOT NULL
+);
+
+CREATE TABLE `app_opens` (
+ `_id` INTEGER PRIMARY KEY,
+ `app_uid` char(40) NOT NULL,
+ `user_id` int(10) NOT NULL,
+ `ts` int(10) NOT NULL,
+ `human_ts` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+
+CREATE TABLE `feedback` (
+ `id` INTEGER PRIMARY KEY,
+ `user_id` int(10) NOT NULL,
+ `message` text,
+ `ts` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE `fsentries` (
+ `id` INTEGER PRIMARY KEY,
+ `uuid` char(36) NOT NULL UNIQUE,
+ `name` varchar(767) NOT NULL,
+ `path` varchar(4096) DEFAULT NULL,
+ `bucket` varchar(50) DEFAULT NULL,
+ `bucket_region` varchar(30) DEFAULT NULL,
+ `public_token` char(36) DEFAULT NULL,
+ `file_request_token` char(36) DEFAULT NULL,
+ `is_shortcut` tinyint(1) DEFAULT '0',
+ `shortcut_to` int(10) DEFAULT NULL,
+ `user_id` int(10) NOT NULL,
+ `parent_id` int(10) DEFAULT NULL,
+ `parent_uid` CHAR(36) NULL DEFAULT NULL,
+ `associated_app_id` int(10) DEFAULT NULL,
+ `is_dir` tinyint(1) DEFAULT '0',
+ `layout` varchar(30) DEFAULT NULL,
+ `sort_by` TEXT DEFAULT NULL,
+ `sort_order` TEXT DEFAULT NULL,
+ `is_public` tinyint(1) DEFAULT NULL,
+ `thumbnail` longtext,
+ `immutable` tinyint(1) NOT NULL DEFAULT '0',
+ `metadata` text,
+ `modified` int(10) NOT NULL,
+ `created` int(10) DEFAULT NULL,
+ `accessed` int(10) DEFAULT NULL,
+ `size` bigint(20) DEFAULT NULL,
+ `symlink_path` varchar(260) DEFAULT NULL,
+ `is_symlink` tinyint(1) DEFAULT '0'
+);
+
+CREATE INDEX idx_parentId_name ON fsentries (`parent_id`, `name`);
+CREATE INDEX idx_path ON fsentries (`path`);
+
+CREATE TABLE `fsentry_versions` (
+ `id` INTEGER PRIMARY KEY,
+ `fsentry_id` int(10) NOT NULL,
+ `fsentry_uuid` char(36) NOT NULL,
+ `version_id` varchar(60) NOT NULL,
+ `user_id` int(10) DEFAULT NULL,
+ `message` mediumtext,
+ `ts_epoch` int(10) DEFAULT NULL
+);
+
+CREATE TABLE `kv` (
+ `id` INTEGER PRIMARY KEY,
+ `app` char(40) DEFAULT NULL,
+ `user_id` int(10) NOT NULL,
+ `kkey_hash` bigint(20) NOT NULL,
+ `kkey` text NOT NULL,
+ `value` text,
+
+ -- 0016
+ `migrated` tinyint(1) DEFAULT '0'
+);
+
+CREATE TABLE `subdomains` (
+ `id` INTEGER PRIMARY KEY,
+ `uuid` varchar(40) DEFAULT NULL,
+ `subdomain` varchar(64) NOT NULL,
+ `user_id` int(10) NOT NULL,
+ `root_dir_id` int(10) DEFAULT NULL,
+ `associated_app_id` int(10) DEFAULT NULL,
+ `ts` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+
+ -- 0015
+ `app_owner` int(10) DEFAULT NULL,
+ FOREIGN KEY (`app_owner`) REFERENCES `apps` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+CREATE TABLE `user` (
+ `id` INTEGER PRIMARY KEY,
+ `uuid` char(36) NOT NULL,
+ `username` varchar(50) DEFAULT NULL,
+ `email` varchar(256) DEFAULT NULL,
+ `password` varchar(225) DEFAULT NULL,
+ `free_storage` bigint(20) DEFAULT NULL,
+ `max_subdomains` int(10) DEFAULT NULL,
+ `taskbar_items` text,
+ `desktop_uuid` CHAR(36) NULL DEFAULT NULL,
+ `appdata_uuid` CHAR(36) NULL DEFAULT NULL,
+ `documents_uuid` CHAR(36) NULL DEFAULT NULL,
+ `pictures_uuid` CHAR(36) NULL DEFAULT NULL,
+ `videos_uuid` CHAR(36) NULL DEFAULT NULL,
+ `trash_uuid` CHAR(36) NULL DEFAULT NULL,
+ `trash_id` INT NULL DEFAULT NULL,
+ `appdata_id` INT NULL DEFAULT NULL,
+ `desktop_id` INT NULL DEFAULT NULL,
+ `documents_id` INT NULL DEFAULT NULL,
+ `pictures_id` INT NULL DEFAULT NULL,
+ `videos_id` INT NULL DEFAULT NULL,
+ `referrer` varchar(64) DEFAULT NULL,
+ `desktop_bg_url` text,
+ `desktop_bg_color` varchar(20) DEFAULT NULL,
+ `desktop_bg_fit` varchar(16) DEFAULT NULL,
+ `pass_recovery_token` char(36) DEFAULT NULL,
+ `requires_email_confirmation` tinyint(1) NOT NULL DEFAULT '0',
+ `email_confirm_code` varchar(8) DEFAULT NULL,
+ `email_confirm_token` char(36) DEFAULT NULL,
+ `email_confirmed` tinyint(1) NOT NULL DEFAULT '0',
+ `dev_first_name` varchar(100) DEFAULT NULL,
+ `dev_last_name` varchar(100) DEFAULT NULL,
+ `dev_paypal` varchar(100) DEFAULT NULL,
+ `dev_approved_for_incentive_program` tinyint(1) DEFAULT '0',
+ `dev_joined_incentive_program` tinyint(1) DEFAULT '0',
+ `suspended` tinyint(1) DEFAULT NULL,
+ `unsubscribed` tinyint(4) NOT NULL DEFAULT '0',
+ `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ `last_activity_ts` timestamp NULL DEFAULT NULL,
+
+ -- 0005
+ `referral_code` VARCHAR(16) DEFAULT NULL,
+ `referred_by` int(10) DEFAULT NULL,
+
+ -- 0007
+ `unconfirmed_change_email` varchar(256) DEFAULT NULL,
+ `change_email_confirm_token` varchar(256) DEFAULT NULL,
+
+ FOREIGN KEY (`referred_by`) REFERENCES `user` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
+
+);
+
+-- 0005
+
+CREATE TABLE `storage_audit` (
+ `id` INTEGER PRIMARY KEY,
+ `user_id` int(10) DEFAULT NULL,
+ `user_id_keep` int(10) NOT NULL,
+ `is_subtract` tinyint(1) NOT NULL DEFAULT '0',
+ `amount` bigint(20) NOT NULL,
+ `field_a` VARCHAR(16) DEFAULT NULL,
+ `field_b` VARCHAR(16) DEFAULT NULL,
+ `reason` VARCHAR(255) DEFAULT NULL,
+ `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- 0008
+
+CREATE TABLE `user_update_audit` (
+ `id` INTEGER PRIMARY KEY,
+ `user_id` int(10) DEFAULT NULL,
+ `user_id_keep` int(10) NOT NULL,
+ `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+ `old_email` varchar(256) DEFAULT NULL,
+ `new_email` varchar(256) DEFAULT NULL,
+ `old_username` varchar(50) DEFAULT NULL,
+ `new_username` varchar(50) DEFAULT NULL,
+
+ -- a message from the service that updated the user's information
+ `reason` VARCHAR(255) DEFAULT NULL,
+
+ FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+CREATE TABLE `app_update_audit` (
+ `id` INTEGER PRIMARY KEY,
+ `app_id` int(10) DEFAULT NULL,
+ `app_id_keep` int(10) NOT NULL,
+ `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+ `old_name` varchar(50) DEFAULT NULL,
+ `new_name` varchar(50) DEFAULT NULL,
+
+ -- a message from the service that updated the app's information
+ `reason` VARCHAR(255) DEFAULT NULL,
+
+ FOREIGN KEY (`app_id`) REFERENCES `apps` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+-- 0009
+
+CREATE TABLE `rl_usage_fixed_window` (
+ `key` varchar(255) NOT NULL,
+ `window_start` bigint NOT NULL,
+ `count` int NOT NULL,
+
+ PRIMARY KEY (`key`)
+);
+
+CREATE TABLE `service_usage_monthly` (
+ `key` varchar(255) NOT NULL,
+ `year` int NOT NULL,
+ `month` int NOT NULL,
+
+ -- these columns are used for querying, so they should also
+ -- be included in the key
+ `user_id` int(10) DEFAULT NULL,
+ `app_id` int(10) DEFAULT NULL,
+
+ `count` int NOT NULL,
+
+ -- 0012
+ `extra` JSON DEFAULT NULL,
+
+ FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE SET NULL ON UPDATE CASCADE,
+ FOREIGN KEY (`app_id`) REFERENCES `apps` (`id`) ON DELETE SET NULL ON UPDATE CASCADE,
+ PRIMARY KEY (`key`, `year`, `month`)
+);
+
+-- 0010
+
+CREATE TABLE `user_to_app_permissions` (
+ `user_id` int(10) NOT NULL,
+ `app_id` int(10) NOT NULL,
+ `permission` varchar(255) NOT NULL,
+ `extra` JSON DEFAULT NULL,
+
+ FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ FOREIGN KEY (`app_id`) REFERENCES `apps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ PRIMARY KEY (`user_id`, `app_id`, `permission`)
+);
+
+CREATE TABLE `audit_user_to_app_permissions` (
+ `id` INTEGER PRIMARY KEY,
+
+ `user_id` int(10) DEFAULT NULL,
+ `user_id_keep` int(10) NOT NULL,
+
+ `app_id` int(10) DEFAULT NULL,
+ `app_id_keep` int(10) NOT NULL,
+
+ `permission` varchar(255) NOT NULL,
+ `extra` JSON DEFAULT NULL,
+
+ `action` VARCHAR(16) DEFAULT NULL, -- "granted" or "revoked"
+ `reason` VARCHAR(255) DEFAULT NULL,
+
+ `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+ FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE SET NULL ON UPDATE CASCADE,
+ FOREIGN KEY (`app_id`) REFERENCES `apps` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+-- 0013
+
+CREATE TABLE `general_analytics` (
+ `id` INTEGER PRIMARY KEY,
+ `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+ `uid` CHAR(40) NOT NULL,
+ `trace_id` VARCHAR(40) DEFAULT NULL,
+ `user_id` int(10) DEFAULT NULL,
+ `user_id_keep` int(10) DEFAULT NULL,
+ `app_id` int(10) DEFAULT NULL,
+ `app_id_keep` int(10) DEFAULT NULL,
+ `server_id` VARCHAR(40) DEFAULT NULL,
+ `actor_type` VARCHAR(40) DEFAULT NULL,
+
+ `tags` JSON,
+ `fields` JSON,
+
+ FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE SET NULL ON UPDATE CASCADE,
+ FOREIGN KEY (`app_id`) REFERENCES `apps` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+-- 0014
+
+CREATE TABLE `auth_audit` (
+ `id` INTEGER PRIMARY KEY,
+ `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+ `uid` CHAR(40) NOT NULL,
+ `ip_address` VARCHAR(45) DEFAULT NULL,
+ `ua_string` VARCHAR(255) DEFAULT NULL,
+
+ `action` VARCHAR(40) DEFAULT NULL,
+
+ `requester` JSON,
+ `body` JSON,
+ `extra` JSON,
+
+ `has_parse_error` TINYINT(1) DEFAULT 0
+
+);
+
+-- 0017
+
+CREATE TABLE `access_token_permissions` (
+ `id` INTEGER PRIMARY KEY,
+ `token_uid` CHAR(40) NOT NULL,
+ `authorizer_user_id` int(10) DEFAULT NULL,
+ `authorizer_app_id` int(10) DEFAULT NULL,
+ `permission` varchar(255) NOT NULL,
+ `extra` JSON DEFAULT NULL,
+ `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP
+
+);
+
+-- 0018
+
+CREATE TABLE `monthly_usage_counts` (
+ `year` int NOT NULL,
+ `month` int NOT NULL,
+ -- what kind of service we're counting
+ `service_type` varchar(40) NOT NULL,
+ -- an identifier in case we offer multiple services of the same type
+ `service_name` varchar(40) NOT NULL,
+ -- an identifier for the actor who is using the service
+ `actor_key` varchar(255) NOT NULL,
+
+ -- the pricing category is a set of values which can be combined
+ -- with locally-fungible values to determine the price of a service
+ `pricing_category` JSON NOT NULL,
+ `pricing_category_hash` binary(20) NOT NULL,
+
+ -- now many times this row has been updated
+ `count` int DEFAULT 0,
+
+ -- values which are locally-fungible within the pricing category
+ `value_uint_1` int DEFAULT NULL,
+ `value_uint_2` int DEFAULT NULL,
+ `value_uint_3` int DEFAULT NULL,
+
+ PRIMARY KEY (
+ `year`, `month`,
+ `service_type`, `service_name`,
+ `actor_key`,
+ `pricing_category_hash`
+ )
+);
diff --git a/packages/backend/src/services/database/sqlite_setup/0002_add-default-apps.sql b/packages/backend/src/services/database/sqlite_setup/0002_add-default-apps.sql
new file mode 100644
index 00000000..5f6deab5
--- /dev/null
+++ b/packages/backend/src/services/database/sqlite_setup/0002_add-default-apps.sql
@@ -0,0 +1,75 @@
+INSERT INTO `apps` (
+ `uid`,
+ `owner_user_id`,
+ `icon`,
+ `name`,
+ `title`,
+ `description`,
+ `index_url`,
+ `approved_for_listing`,
+ `approved_for_opening_items`,
+ `approved_for_incentive_program`,
+ `timestamp`,
+ `last_review`
+) VALUES (
+ 'app-838dfbc4-bf8b-48c2-b47b-c4adc77fab58',
+ 1,
+ 'data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjIiIGJhc2VQcm9maWxlPSJ0aW55LXBzIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0OCA0OCIgd2lkdGg9IjQ4IiBoZWlnaHQ9IjQ4Ij4KCTx0aXRsZT5hcHAtaWNvbi1lZGl0b3Itc3ZnPC90aXRsZT4KCTxkZWZzPgoJCTxsaW5lYXJHcmFkaWVudCBpZD0iZ3JkMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiICB4MT0iNDciIHkxPSIzOS41MTQiIHgyPSIxIiB5Mj0iOC40ODYiPgoJCQk8c3RvcCBvZmZzZXQ9IjAiIHN0b3AtY29sb3I9IiM3MTAxZTgiICAvPgoJCQk8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM5MTY3YmUiICAvPgoJCTwvbGluZWFyR3JhZGllbnQ+Cgk8L2RlZnM+Cgk8c3R5bGU+CgkJdHNwYW4geyB3aGl0ZS1zcGFjZTpwcmUgfQoJCS5zaHAwIHsgZmlsbDogdXJsKCNncmQxKSB9IAoJCS5zaHAxIHsgZmlsbDogI2ZmZmZmZiB9IAoJPC9zdHlsZT4KCTxnIGlkPSJMYXllciI+CgkJPHBhdGggaWQ9IkxheWVyIiBjbGFzcz0ic2hwMCIgZD0iTTQ3IDNMNDcgNDVDNDcgNDYuMSA0Ni4xIDQ3IDQ1IDQ3TDMgNDdDMS45IDQ3IDEgNDYuMSAxIDQ1TDEgM0MxIDEuOSAxLjkgMSAzIDFMNDUgMUM0Ni4xIDEgNDcgMS45IDQ3IDNaIiAvPgoJCTxwYXRoIGlkPSJMYXllciIgZmlsbC1ydWxlPSJldmVub2RkIiBjbGFzcz0ic2hwMSIgZD0iTTI4LjYyIDQwTDI4LjYyIDM3LjYxTDMyLjI1IDM3LjIyTDI5Ljg2IDMwTDE3LjUzIDMwTDE1LjE4IDM3LjIyTDE4Ljc2IDM3LjYxTDE4Ljc2IDQwTDguNiA0MEw4LjYgMzcuNjZMMTAuNSAzNy4xN0MxMS4yMSAzNi45OSAxMS40MyAzNi44NiAxMS42IDM2LjMzTDIxLjMzIDhMMjYuNDUgOEwzNi4zNiAzNi4zOEMzNi41MyAzNi45MSAzNi44OCAzNi45OSAzNy40MiAzNy4xM0wzOS40IDM3LjYxTDM5LjQgNDBMMjguNjIgNDBaTTIzLjc2IDExLjQ1TDE4LjU0IDI3TDI4Ljg4IDI3TDIzLjc2IDExLjQ1WiIgLz4KCTwvZz4KPC9zdmc+',
+ 'editor',
+ 'Editor',
+ 'A simple text editor',
+ 'https://editor.puter.com/index.html',
+ 1, 1, 0,
+ '2020-01-01 00:00:00',
+ NULL
+);
+
+INSERT INTO `apps` (
+ `uid`,
+ `owner_user_id`,
+ `icon`,
+ `name`,
+ `title`,
+ `description`,
+ `index_url`,
+ `approved_for_listing`,
+ `approved_for_opening_items`,
+ `approved_for_incentive_program`,
+ `timestamp`,
+ `last_review`,
+ `godmode`
+) VALUES (
+ 'app-3fea7529-266e-47d9-8776-31649cd06557',
+ 1,
+ 'data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPHN2ZyBzdHlsZT0iZmlsdGVyOiBkcm9wLXNoYWRvdyggMHB4IDFweCAxcHggcmdiYSgwLCAwLCAwLCAuNSkpOyIgaGVpZ2h0PSI0OCIgd2lkdGg9IjQ4IiB2aWV3Qm94PSIwIDAgNDggNDgiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CiAgPHRpdGxlPndpbmRvdyBjb2RlPC90aXRsZT4KICA8ZyBjbGFzcz0ibmMtaWNvbi13cmFwcGVyIiBzdHlsZT0iIiB0cmFuc2Zvcm09Im1hdHJpeCgwLjk5NzcyNiwgMCwgMCwgMS4xMDI3NDgsIC0wLjAwMjc5MSwgLTIuODA5NzIxKSI+CiAgICA8cGF0aCBkPSJNIDQ1LjA5OCA0NS4zNjIgTCAzLjAwNCA0NS4zNjIgQyAxLjg5NyA0NS4zNjIgMSA0NC40NTkgMSA0My4zNDUgTCAxIDUuMDE3IEMgMSAzLjkwMyAxLjg5NyAzIDMuMDA0IDMgTCA0NS4wOTggMyBDIDQ2LjIwNiAzIDQ3LjEwMyAzLjkwMyA0Ny4xMDMgNS4wMTcgTCA0Ny4xMDMgNDMuMzQ1IEMgNDcuMTAzIDQ0LjQ1OSA0Ni4yMDYgNDUuMzYyIDQ1LjA5OCA0NS4zNjIgWiIgc3R5bGU9ImZpbGwtcnVsZTogbm9uemVybzsgcGFpbnQtb3JkZXI6IGZpbGw7IiBmaWxsPSIjZTNlNWVjIi8+CiAgICA8cmVjdCB4PSIzLjAwNCIgeT0iMTAuMDYiIGZpbGw9IiMyZTM3NDQiIHdpZHRoPSI0Mi4wOTQiIGhlaWdodD0iMzMuMjg0IiBzdHlsZT0iIi8+CiAgICA8cGF0aCBmaWxsPSIjRkZGRkZGIiBkPSJNIDEwLjAyIDMxLjI0MSBDIDkuNzY0IDMxLjI0MSA5LjUwNyAzMS4xNDIgOS4zMTIgMzAuOTQ2IEMgOC45MiAzMC41NTEgOC45MiAyOS45MTQgOS4zMTIgMjkuNTIgTCAxMi42MTIgMjYuMTk4IEwgOS4zMTIgMjIuODc3IEMgOC45MiAyMi40ODIgOC45MiAyMS44NDUgOS4zMTIgMjEuNDUxIEMgOS43MDMgMjEuMDU2IDEwLjMzNyAyMS4wNTYgMTAuNzI5IDIxLjQ1MSBMIDE0LjczOCAyNS40ODUgQyAxNS4xMyAyNS44NzkgMTUuMTMgMjYuNTE3IDE0LjczOCAyNi45MTEgTCAxMC43MjkgMzAuOTQ2IEMgMTAuNTMzIDMxLjE0MiAxMC4yNzcgMzEuMjQxIDEwLjAyIDMxLjI0MSBaIiBzdHlsZT0iIi8+CiAgICA8cGF0aCBmaWxsPSIjRkZGRkZGIiBkPSJNIDI4LjA2IDMxLjI0MSBMIDIwLjA0MyAzMS4yNDEgQyAxOS40ODkgMzEuMjQxIDE5LjA0IDMwLjc4OSAxOS4wNCAzMC4yMzMgQyAxOS4wNCAyOS42NzYgMTkuNDg5IDI5LjIyNCAyMC4wNDMgMjkuMjI0IEwgMjguMDYgMjkuMjI0IEMgMjguNjE0IDI5LjIyNCAyOS4wNjMgMjkuNjc2IDI5LjA2MyAzMC4yMzMgQyAyOS4wNjMgMzAuNzg5IDI4LjYxNCAzMS4yNDEgMjguMDYgMzEuMjQxIFoiIHN0eWxlPSIiLz4KICA8L2c+Cjwvc3ZnPg==',
+ 'terminal',
+ 'Terminal',
+ 'A simple terminal',
+ 'https://puter.sh',
+ 1, 1, 0,
+ '2020-01-01 00:00:00',
+ NULL,
+ 1
+);
+
+INSERT INTO `apps` (
+ `id`, `uid`, `owner_user_id`, `icon`, `name`, `title`, `description`, `godmode`, `maximize_on_start`, `index_url`, `approved_for_listing`, `approved_for_opening_items`, `approved_for_incentive_program`, `timestamp`, `last_review`, `tags`, `app_owner`
+) VALUES (14,'app-7870be61-8dff-4a99-af64-e9ae6811e367',60950,
+ 'data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjIiIGJhc2VQcm9maWxlPSJ0aW55LXBzIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0OCA0OCIgd2lkdGg9IjQ4IiBoZWlnaHQ9IjQ4Ij4KCTx0aXRsZT5hcHAtaWNvbi12aWV3ZXItc3ZnPC90aXRsZT4KCTxkZWZzPgoJCTxsaW5lYXJHcmFkaWVudCBpZD0iZ3JkMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiICB4MT0iNDciIHkxPSIzOS41MTQiIHgyPSIxIiB5Mj0iOC40ODYiPgoJCQk8c3RvcCBvZmZzZXQ9IjAiIHN0b3AtY29sb3I9IiMwMzYzYWQiICAvPgoJCQk8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM1Njg0ZjUiICAvPgoJCTwvbGluZWFyR3JhZGllbnQ+Cgk8L2RlZnM+Cgk8c3R5bGU+CgkJdHNwYW4geyB3aGl0ZS1zcGFjZTpwcmUgfQoJCS5zaHAwIHsgZmlsbDogdXJsKCNncmQxKSB9IAoJCS5zaHAxIHsgZmlsbDogI2ZmZDc2NCB9IAoJCS5zaHAyIHsgZmlsbDogI2NiZWFmYiB9IAoJPC9zdHlsZT4KCTxnIGlkPSJMYXllciI+CgkJPHBhdGggaWQ9IlNoYXBlIDEiIGNsYXNzPSJzaHAwIiBkPSJNMSAxTDQ3IDFMNDcgNDdMMSA0N0wxIDFaIiAvPgoJCTxwYXRoIGlkPSJMYXllciIgY2xhc3M9InNocDEiIGQ9Ik0xOCAxOEMxNS43OSAxOCAxNCAxNi4yMSAxNCAxNEMxNCAxMS43OSAxNS43OSAxMCAxOCAxMEMyMC4yMSAxMCAyMiAxMS43OSAyMiAxNEMyMiAxNi4yMSAyMC4yMSAxOCAxOCAxOFoiIC8+CgkJPHBhdGggaWQ9IkxheWVyIiBjbGFzcz0ic2hwMiIgZD0iTTM5Ljg2IDM2LjUxQzM5LjgyIDM2LjU4IDM5Ljc3IDM2LjY1IDM5LjcgMzYuNzFDMzkuNjQgMzYuNzcgMzkuNTcgMzYuODIgMzkuNSAzNi44N0MzOS40MiAzNi45MSAzOS4zNCAzNi45NCAzOS4yNiAzNi45N0MzOS4xNyAzNi45OSAzOS4wOSAzNyAzOSAzN0w5IDM3QzguODIgMzcgOC42NCAzNi45NSA4LjQ5IDM2Ljg2QzguMzMgMzYuNzYgOC4yIDM2LjYzIDguMTIgMzYuNDdDOC4wMyAzNi4zMSA3Ljk5IDM2LjEzIDggMzUuOTVDOC4wMSAzNS43NyA4LjA3IDM1LjYgOC4xNyAzNS40NEwxNC4xNyAyNi40NUMxNC4yNCAyNi4zNCAxNC4zMyAyNi4yNCAxNC40NCAyNi4xN0MxNC41NSAyNi4xIDE0LjY4IDI2LjA0IDE0LjggMjYuMDJDMTQuOTMgMjUuOTkgMTUuMDcgMjUuOTkgMTUuMTkgMjYuMDJDMTUuMzIgMjYuMDQgMTUuNDUgMjYuMSAxNS41NSAyNi4xN0MxNS41NyAyNi4xOCAxNS41OCAyNi4xOSAxNS42IDI2LjJDMTUuNjEgMjYuMjEgMTUuNjIgMjYuMjIgMTUuNjMgMjYuMjNDMTUuNjUgMjYuMjQgMTUuNjYgMjYuMjUgMTUuNjcgMjYuMjZDMTUuNjggMjYuMjcgMTUuNyAyNi4yOCAxNS43MSAyNi4yOUwyMC44NiAzMS40NUwyOS4xOCAxOS40M0MyOS4yMyAxOS4zNiAyOS4yOCAxOS4zIDI5LjM1IDE5LjI0QzI5LjQxIDE5LjE5IDI5LjQ4IDE5LjE0IDI5LjU2IDE5LjFDMjkuNjMgMTkuMDYgMjkuNzEgMTkuMDQgMjkuNzkgMTkuMDJDMjkuODggMTkgMjkuOTYgMTkgMzAuMDUgMTlDMzAuMTMgMTkgMzAuMjEgMTkuMDIgMzAuMjkgMTkuMDRDMzAuMzggMTkuMDcgMzAuNDUgMTkuMSAzMC41MiAxOS4xNUMzMC42IDE5LjE5IDMwLjY2IDE5LjI1IDMwLjcyIDE5LjMxQzMwLjc4IDE5LjM3IDMwLjgzIDE5LjQ0IDMwLjg3IDE5LjUxTDM5Ljg3IDM1LjUxQzM5LjkxIDM1LjU5IDM5Ljk1IDM1LjY3IDM5Ljk3IDM1Ljc1QzM5Ljk5IDM1Ljg0IDQwIDM1LjkyIDQwIDM2LjAxQzQwIDM2LjEgMzkuOTkgMzYuMTggMzkuOTYgMzYuMjdDMzkuOTQgMzYuMzUgMzkuOTEgMzYuNDMgMzkuODYgMzYuNTFaIiAvPgoJPC9nPgo8L3N2Zz4=',
+ 'viewer','Viewer','',0,1,'https://viewer.puter.com/index.html',1,0,0,'2022-08-16 01:40:02',NULL,NULL,NULL);
+
+INSERT INTO `apps` (`id`, `uid`, `owner_user_id`, `icon`, `name`, `title`, `description`, `godmode`, `maximize_on_start`, `index_url`, `approved_for_listing`, `approved_for_opening_items`, `approved_for_incentive_program`, `timestamp`, `last_review`, `tags`, `app_owner`) VALUES (6,'app-3920851d-bda8-479b-9407-8517293c7d44',60950,
+ 'data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjAuMCwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPCFET0NUWVBFIHN2ZyBQVUJMSUMgIi0vL1czQy8vRFREIFNWRyAxLjEvL0VOIiAiaHR0cDovL3d3dy53My5vcmcvR3JhcGhpY3MvU1ZHLzEuMS9EVEQvc3ZnMTEuZHRkIj4NCjxzdmcgdmVyc2lvbj0iMS4xIiBpZD0iQ2FwYV8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDU2IDU2IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA1NiA1NjsiIHhtbDpzcGFjZT0icHJlc2VydmUiPg0KPGc+DQoJPHBhdGggc3R5bGU9ImZpbGw6I0U5RTlFMDsiIGQ9Ik0zNi45ODUsMEg3Ljk2M0M3LjE1NSwwLDYuNSwwLjY1NSw2LjUsMS45MjZWNTVjMCwwLjM0NSwwLjY1NSwxLDEuNDYzLDFoNDAuMDc0DQoJCWMwLjgwOCwwLDEuNDYzLTAuNjU1LDEuNDYzLTFWMTIuOTc4YzAtMC42OTYtMC4wOTMtMC45Mi0wLjI1Ny0xLjA4NUwzNy42MDcsMC4yNTdDMzcuNDQyLDAuMDkzLDM3LjIxOCwwLDM2Ljk4NSwweiIvPg0KCTxwb2x5Z29uIHN0eWxlPSJmaWxsOiNEOUQ3Q0E7IiBwb2ludHM9IjM3LjUsMC4xNTEgMzcuNSwxMiA0OS4zNDksMTIgCSIvPg0KCTxwYXRoIHN0eWxlPSJmaWxsOiNDQzRCNEM7IiBkPSJNMTkuNTE0LDMzLjMyNEwxOS41MTQsMzMuMzI0Yy0wLjM0OCwwLTAuNjgyLTAuMTEzLTAuOTY3LTAuMzI2DQoJCWMtMS4wNDEtMC43ODEtMS4xODEtMS42NS0xLjExNS0yLjI0MmMwLjE4Mi0xLjYyOCwyLjE5NS0zLjMzMiw1Ljk4NS01LjA2OGMxLjUwNC0zLjI5NiwyLjkzNS03LjM1NywzLjc4OC0xMC43NQ0KCQljLTAuOTk4LTIuMTcyLTEuOTY4LTQuOTktMS4yNjEtNi42NDNjMC4yNDgtMC41NzksMC41NTctMS4wMjMsMS4xMzQtMS4yMTVjMC4yMjgtMC4wNzYsMC44MDQtMC4xNzIsMS4wMTYtMC4xNzINCgkJYzAuNTA0LDAsMC45NDcsMC42NDksMS4yNjEsMS4wNDljMC4yOTUsMC4zNzYsMC45NjQsMS4xNzMtMC4zNzMsNi44MDJjMS4zNDgsMi43ODQsMy4yNTgsNS42Miw1LjA4OCw3LjU2Mg0KCQljMS4zMTEtMC4yMzcsMi40MzktMC4zNTgsMy4zNTgtMC4zNThjMS41NjYsMCwyLjUxNSwwLjM2NSwyLjkwMiwxLjExN2MwLjMyLDAuNjIyLDAuMTg5LDEuMzQ5LTAuMzksMi4xNg0KCQljLTAuNTU3LDAuNzc5LTEuMzI1LDEuMTkxLTIuMjIsMS4xOTFjLTEuMjE2LDAtMi42MzItMC43NjgtNC4yMTEtMi4yODVjLTIuODM3LDAuNTkzLTYuMTUsMS42NTEtOC44MjgsMi44MjINCgkJYy0wLjgzNiwxLjc3NC0xLjYzNywzLjIwMy0yLjM4Myw0LjI1MUMyMS4yNzMsMzIuNjU0LDIwLjM4OSwzMy4zMjQsMTkuNTE0LDMzLjMyNHogTTIyLjE3NiwyOC4xOTgNCgkJYy0yLjEzNywxLjIwMS0zLjAwOCwyLjE4OC0zLjA3MSwyLjc0NGMtMC4wMSwwLjA5Mi0wLjAzNywwLjMzNCwwLjQzMSwwLjY5MkMxOS42ODUsMzEuNTg3LDIwLjU1NSwzMS4xOSwyMi4xNzYsMjguMTk4eg0KCQkgTTM1LjgxMywyMy43NTZjMC44MTUsMC42MjcsMS4wMTQsMC45NDQsMS41NDcsMC45NDRjMC4yMzQsMCwwLjkwMS0wLjAxLDEuMjEtMC40NDFjMC4xNDktMC4yMDksMC4yMDctMC4zNDMsMC4yMy0wLjQxNQ0KCQljLTAuMTIzLTAuMDY1LTAuMjg2LTAuMTk3LTEuMTc1LTAuMTk3QzM3LjEyLDIzLjY0OCwzNi40ODUsMjMuNjcsMzUuODEzLDIzLjc1NnogTTI4LjM0MywxNy4xNzQNCgkJYy0wLjcxNSwyLjQ3NC0xLjY1OSw1LjE0NS0yLjY3NCw3LjU2NGMyLjA5LTAuODExLDQuMzYyLTEuNTE5LDYuNDk2LTIuMDJDMzAuODE1LDIxLjE1LDI5LjQ2NiwxOS4xOTIsMjguMzQzLDE3LjE3NHoNCgkJIE0yNy43MzYsOC43MTJjLTAuMDk4LDAuMDMzLTEuMzMsMS43NTcsMC4wOTYsMy4yMTZDMjguNzgxLDkuODEzLDI3Ljc3OSw4LjY5OCwyNy43MzYsOC43MTJ6Ii8+DQoJPHBhdGggc3R5bGU9ImZpbGw6I0NDNEI0QzsiIGQ9Ik00OC4wMzcsNTZINy45NjNDNy4xNTUsNTYsNi41LDU1LjM0NSw2LjUsNTQuNTM3VjM5aDQzdjE1LjUzN0M0OS41LDU1LjM0NSw0OC44NDUsNTYsNDguMDM3LDU2eiIvPg0KCTxnPg0KCQk8cGF0aCBzdHlsZT0iZmlsbDojRkZGRkZGOyIgZD0iTTE3LjM4NSw1M2gtMS42NDFWNDIuOTI0aDIuODk4YzAuNDI4LDAsMC44NTIsMC4wNjgsMS4yNzEsMC4yMDUNCgkJCWMwLjQxOSwwLjEzNywwLjc5NSwwLjM0MiwxLjEyOCwwLjYxNWMwLjMzMywwLjI3MywwLjYwMiwwLjYwNCwwLjgwNywwLjk5MXMwLjMwOCwwLjgyMiwwLjMwOCwxLjMwNg0KCQkJYzAsMC41MTEtMC4wODcsMC45NzMtMC4yNiwxLjM4OGMtMC4xNzMsMC40MTUtMC40MTUsMC43NjQtMC43MjUsMS4wNDZjLTAuMzEsMC4yODItMC42ODQsMC41MDEtMS4xMjEsMC42NTYNCgkJCXMtMC45MjEsMC4yMzItMS40NDksMC4yMzJoLTEuMjE3VjUzeiBNMTcuMzg1LDQ0LjE2OHYzLjk5MmgxLjUwNGMwLjIsMCwwLjM5OC0wLjAzNCwwLjU5NS0wLjEwMw0KCQkJYzAuMTk2LTAuMDY4LDAuMzc2LTAuMTgsMC41NC0wLjMzNWMwLjE2NC0wLjE1NSwwLjI5Ni0wLjM3MSwwLjM5Ni0wLjY0OWMwLjEtMC4yNzgsMC4xNS0wLjYyMiwwLjE1LTEuMDMyDQoJCQljMC0wLjE2NC0wLjAyMy0wLjM1NC0wLjA2OC0wLjU2N2MtMC4wNDYtMC4yMTQtMC4xMzktMC40MTktMC4yOC0wLjYxNWMtMC4xNDItMC4xOTYtMC4zNC0wLjM2LTAuNTk1LTAuNDkyDQoJCQljLTAuMjU1LTAuMTMyLTAuNTkzLTAuMTk4LTEuMDEyLTAuMTk4SDE3LjM4NXoiLz4NCgkJPHBhdGggc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIGQ9Ik0zMi4yMTksNDcuNjgyYzAsMC44MjktMC4wODksMS41MzgtMC4yNjcsMi4xMjZzLTAuNDAzLDEuMDgtMC42NzcsMS40NzdzLTAuNTgxLDAuNzA5LTAuOTIzLDAuOTM3DQoJCQlzLTAuNjcyLDAuMzk4LTAuOTkxLDAuNTEzYy0wLjMxOSwwLjExNC0wLjYxMSwwLjE4Ny0wLjg3NSwwLjIxOUMyOC4yMjIsNTIuOTg0LDI4LjAyNiw1MywyNy44OTgsNTNoLTMuODE0VjQyLjkyNGgzLjAzNQ0KCQkJYzAuODQ4LDAsMS41OTMsMC4xMzUsMi4yMzUsMC40MDNzMS4xNzYsMC42MjcsMS42LDEuMDczczAuNzQsMC45NTUsMC45NSwxLjUyNEMzMi4xMTQsNDYuNDk0LDMyLjIxOSw0Ny4wOCwzMi4yMTksNDcuNjgyeg0KCQkJIE0yNy4zNTIsNTEuNzk3YzEuMTEyLDAsMS45MTQtMC4zNTUsMi40MDYtMS4wNjZzMC43MzgtMS43NDEsMC43MzgtMy4wOWMwLTAuNDE5LTAuMDUtMC44MzQtMC4xNS0xLjI0NA0KCQkJYy0wLjEwMS0wLjQxLTAuMjk0LTAuNzgxLTAuNTgxLTEuMTE0cy0wLjY3Ny0wLjYwMi0xLjE2OS0wLjgwN3MtMS4xMy0wLjMwOC0xLjkxNC0wLjMwOGgtMC45NTd2Ny42MjlIMjcuMzUyeiIvPg0KCQk8cGF0aCBzdHlsZT0iZmlsbDojRkZGRkZGOyIgZD0iTTM2LjI2Niw0NC4xNjh2My4xNzJoNC4yMTF2MS4xMjFoLTQuMjExVjUzaC0xLjY2OFY0Mi45MjRINDAuOXYxLjI0NEgzNi4yNjZ6Ii8+DQoJPC9nPg0KPC9nPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPC9zdmc+DQo=',
+ 'pdf','PDF','',0,1,'https://pdf.puter.com/index.html',1,0,0,'2022-08-16 01:28:47',NULL,'productivity',NULL);
+
+INSERT INTO `apps` (`id`, `uid`, `owner_user_id`, `icon`, `name`, `title`, `description`, `godmode`, `maximize_on_start`, `index_url`, `approved_for_listing`, `approved_for_opening_items`, `approved_for_incentive_program`, `timestamp`, `last_review`, `tags`, `app_owner`) VALUES (9,'app-5584fbf7-ed69-41fc-99cd-85da21b1ef51',60950,
+ 'data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPHN2ZyB2aWV3Qm94PSIwIDAgNTEyIDUxMiIgd2lkdGg9IjUxMiIgaGVpZ2h0PSI1MTIiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CiAgPGRlZnM+CiAgICA8bGluZWFyR3JhZGllbnQgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIHgxPSIyNTYiIHkxPSIwIiB4Mj0iMjU2IiB5Mj0iNTEyIiBpZD0iZ3JhZGllbnQtMCI+CiAgICAgIDxzdG9wIG9mZnNldD0iMCIgc3R5bGU9InN0b3AtY29sb3I6IHJnYigwLCAxMiwgMTA4KTsiLz4KICAgICAgPHN0b3Agb2Zmc2V0PSIxIiBzdHlsZT0ic3RvcC1jb2xvcjogcmdiKDE2LCAwLCAxNDkpOyIvPgogICAgPC9saW5lYXJHcmFkaWVudD4KICA8L2RlZnM+CiAgPHJlY3Qgc3R5bGU9InBhaW50LW9yZGVyOiBmaWxsOyBmaWxsLXJ1bGU6IG5vbnplcm87IGZpbGw6IHVybCgnI2dyYWRpZW50LTAnKTsiIHg9IjAiIHk9IjAiIHdpZHRoPSI1MTIiIGhlaWdodD0iNTEyIiByeD0iNzAiIHJ5PSI3MCIvPgogIDxjaXJjbGUgY3g9IjE3OC4zMzciIGN5PSIyNTguODc2IiBmaWxsPSIjYzBkYWRjIiByPSIyOSIgc3R5bGU9IiIgdHJhbnNmb3JtPSJtYXRyaXgoNi4xMDExMTEsIDAsIDAsIDYuMTI2OTY2LCAtODMzLjU4ODg2NywgLTEzMzAuODY4MDQyKSIvPgogIDxjaXJjbGUgY3g9IjE3OC4zMzciIGN5PSIyNTguODc2IiBmaWxsPSIjNGQ2ZmM0IiByPSIyMyIgc3R5bGU9IiIgdHJhbnNmb3JtPSJtYXRyaXgoNi4xMDExMTEsIDAsIDAsIDYuMTI2OTY2LCAtODMzLjU4ODg2NywgLTEzMzAuODY4MDQyKSIvPgogIDxjaXJjbGUgY3g9IjE3OC4zMzciIGN5PSIyNTguODc2IiBmaWxsPSIjM2Q1ZmEzIiByPSIxOCIgc3R5bGU9IiIgdHJhbnNmb3JtPSJtYXRyaXgoNi4xMDExMTEsIDAsIDAsIDYuMTI2OTY2LCAtODMzLjU4ODg2NywgLTEzMzAuODY4MDQyKSIvPgogIDxwYXRoIGQ9Ik0gMjExLjAyNSAxODguNjU2IEMgMjYyLjE0NiAxNTUuMDA2IDMzMC4zNzQgMTg5LjU1IDMzMy44MzQgMjUwLjgzOCBDIDMzNy4yOTMgMzEyLjEyNyAyNzMuMzkgMzU0LjE4OSAyMTguODA5IDMyNi41NTUgQyAxNzYuNDc0IDMwNS4xMjMgMTYyLjE1NSAyNTEuNDUxIDE4OC4xNDYgMjExLjYzMiBMIDIxMS4wMjUgMTg4LjY1NiBaIiBmaWxsPSIjMmY0Yjc3IiBzdHlsZT0iIi8+CiAgPGcgZmlsbD0iI2ZmZiIgdHJhbnNmb3JtPSJtYXRyaXgoNi4xMDExMTEsIDAsIDAsIDYuMTI2OTY2LCA3MS40MzIxOSwgNzEuNDQ5NjIzKSIgc3R5bGU9IiI+CiAgICA8Y2lyY2xlIGN4PSIyNCIgY3k9IjI0IiByPSI1Ii8+CiAgICA8Y2lyY2xlIGN4PSIzMi41IiBjeT0iMzIuNSIgcj0iMi41Ii8+CiAgPC9nPgo8L3N2Zz4=',
+ 'camera','Camera','Camera in the browser.',0,0,'https://camera.puter.com/index.html',1,0,0,'2022-08-16 01:32:36',NULL,NULL,NULL);
+
+INSERT INTO `apps` (`id`, `uid`, `owner_user_id`, `icon`, `name`, `title`, `description`, `godmode`, `maximize_on_start`, `index_url`, `approved_for_listing`, `approved_for_opening_items`, `approved_for_incentive_program`, `timestamp`, `last_review`, `tags`, `app_owner`) VALUES (5,'app-11edfba2-1ed3-4e22-8573-47e88fb87d70',60950,
+ 'data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE5LjAuMCwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDUxMi4wMDEgNTEyLjAwMSIgc3R5bGU9ImVuYWJsZS1iYWNrZ3JvdW5kOm5ldyAwIDAgNTEyLjAwMSA1MTIuMDAxOyIgeG1sOnNwYWNlPSJwcmVzZXJ2ZSI+DQo8cGF0aCBzdHlsZT0iZmlsbDojNTE1MDRFOyIgZD0iTTQ5MC42NjUsNDMuNTU3SDIxLjMzM0M5LjU1Miw0My41NTcsMCw1My4xMDgsMCw2NC44OXYzODIuMjJjMCwxMS43ODIsOS41NTIsMjEuMzM0LDIxLjMzMywyMS4zMzQNCgloNDY5LjMzMmMxMS43ODMsMCwyMS4zMzUtOS41NTIsMjEuMzM1LTIxLjMzNFY2NC44OUM1MTIsNTMuMTA4LDUwMi40NDgsNDMuNTU3LDQ5MC42NjUsNDMuNTU3eiBNOTkuMDMsNDI3LjA1MUg1Ni4yNjd2LTM4LjA2OQ0KCUg5OS4wM1Y0MjcuMDUxeiBNOTkuMDMsMTIzLjAxOUg1Ni4yNjd2LTM4LjA3SDk5LjAzVjEyMy4wMTl6IE0xODguMjA2LDQyNy4wNTFoLTQyLjc2M3YtMzguMDY5aDQyLjc2M1Y0MjcuMDUxeiBNMTg4LjIwNiwxMjMuMDE5DQoJaC00Mi43NjN2LTM4LjA3aDQyLjc2M1YxMjMuMDE5eiBNMjc3LjM4Miw0MjcuMDUxaC00Mi43NjR2LTM4LjA2OWg0Mi43NjRWNDI3LjA1MXogTTI3Ny4zODIsMTIzLjAxOWgtNDIuNzY0di0zOC4wN2g0Mi43NjRWMTIzLjAxOQ0KCXogTTM2Ni41NTcsNDI3LjA1MWgtNDIuNzYzdi0zOC4wNjloNDIuNzYzVjQyNy4wNTF6IE0zNjYuNTU3LDEyMy4wMTloLTQyLjc2M3YtMzguMDdoNDIuNzYzVjEyMy4wMTl6IE00NTUuNzMzLDQyNy4wNTFINDEyLjk3DQoJdi0zOC4wNjloNDIuNzY0djM4LjA2OUg0NTUuNzMzeiBNNDU1LjczMywxMjMuMDE5SDQxMi45N3YtMzguMDdoNDIuNzY0djM4LjA3SDQ1NS43MzN6Ii8+DQo8cGF0aCBzdHlsZT0iZmlsbDojNkI2OTY4OyIgZD0iTTQ5MC42NjUsNDMuNTU3SDEzMy44MWMtMTYuMzQzLDM4Ljg3Ny0yNS4zODEsODEuNTgtMjUuMzgxLDEyNi4zOTYNCgljMCwxMzMuMTkyLDc5Ljc4MiwyNDcuNzM0LDE5NC4xNTUsMjk4LjQ5aDE4OC4wODJjMTEuNzgzLDAsMjEuMzM1LTkuNTUyLDIxLjMzNS0yMS4zMzRWNjQuODkNCglDNTEyLDUzLjEwOCw1MDIuNDQ4LDQzLjU1Nyw0OTAuNjY1LDQzLjU1N3ogTTE4OC4yMDYsMTIzLjAxOWgtNDIuNzYzdi0zOC4wN2g0Mi43NjNWMTIzLjAxOXogTTI3Ny4zODIsNDI3LjA1MWgtNDIuNzY0di0zOC4wNjkNCgloNDIuNzY0VjQyNy4wNTF6IE0yNzcuMzgyLDEyMy4wMTloLTQyLjc2NHYtMzguMDdoNDIuNzY0VjEyMy4wMTl6IE0zNjYuNTU3LDQyNy4wNTFoLTQyLjc2M3YtMzguMDY5aDQyLjc2M1Y0MjcuMDUxeg0KCSBNMzY2LjU1NywxMjMuMDE5aC00Mi43NjN2LTM4LjA3aDQyLjc2M1YxMjMuMDE5eiBNNDU1LjczMyw0MjcuMDUxSDQxMi45N3YtMzguMDY5aDQyLjc2NHYzOC4wNjlINDU1LjczM3ogTTQ1NS43MzMsMTIzLjAxOUg0MTIuOTcNCgl2LTM4LjA3aDQyLjc2NHYzOC4wN0g0NTUuNzMzeiIvPg0KPHBhdGggc3R5bGU9ImZpbGw6Izg4RENFNTsiIGQ9Ik0zMTguNjEyLDI0My42NTdsLTExMi44OC01Ni40NGMtOS4xOTEtNC41OTUtMTkuOTc0LDIuMTMtMTkuOTc0LDEyLjM0NlYzMTIuNDQNCgljMCwxMC4yNjcsMTAuODM3LDE2LjkyNywxOS45NzQsMTIuMzQ1bDExMi44OC01Ni40MzljNC42NzQtMi4zMzgsNy42MjgtNy4xMTcsNy42MjgtMTIuMzQ1DQoJQzMyNi4yNCwyNTAuNzc0LDMyMy4yODYsMjQ1Ljk5NSwzMTguNjEyLDI0My42NTd6Ii8+DQo8cGF0aCBzdHlsZT0iZmlsbDojNzRDNEM0OyIgZD0iTTIxMS41MTUsMTk5LjU2MmMwLTIuOTY4LDAuOTU3LTUuODAyLDIuNjUyLTguMTI4bC04LjQzNS00LjIxOA0KCWMtOS4xOTEtNC41OTUtMTkuOTc0LDIuMTMtMTkuOTc0LDEyLjM0NlYzMTIuNDRjMCwxMC4yNjcsMTAuODM3LDE2LjkyNywxOS45NzQsMTIuMzQ1bDguNDMzLTQuMjE3DQoJQzIxMC41MDgsMzE1LjU0NywyMTEuNTE1LDMyMS45NjksMjExLjUxNSwxOTkuNTYyeiIvPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPGc+DQo8L2c+DQo8Zz4NCjwvZz4NCjxnPg0KPC9nPg0KPC9zdmc+DQo=',
+ 'player','Player','A free video player app in the browser.',0,0,'https://player.puter.com/index.html',1,0,0,'2022-08-16 01:27:30',NULL,NULL,NULL);
+
+INSERT INTO `apps` (`id`, `uid`, `owner_user_id`, `icon`, `name`, `title`, `description`, `godmode`, `maximize_on_start`, `index_url`, `approved_for_listing`, `approved_for_opening_items`, `approved_for_incentive_program`, `timestamp`, `last_review`, `tags`, `app_owner`) VALUES (562,'app-7bdca1a4-6373-4c98-ad97-03ff2d608ca1',60950,
+ 'data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjIiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgdmlld0JveD0iMCAwIDUxMiA1MTIiIHdpZHRoPSI1MTIiIGhlaWdodD0iNTEyIj48ZGVmcz48aW1hZ2UgIHdpZHRoPSIzNjEiIGhlaWdodD0iMzYxIiBpZD0iaW1nMSIgaHJlZj0iZGF0YTppbWFnZS9wbmc7YmFzZTY0LGlWQk9SdzBLR2dvQUFBQU5TVWhFVWdBQUFXa0FBQUZwQVFNQUFBQmt0VXNOQUFBQUFYTlNSMElCMmNrc2Z3QUFBQU5RVEZSRi8vLy9wOFFieUFBQUFDZEpSRUZVZUp6dHdRRU5BQUFBd3FEM1QyMFBCeFFBQUFBQUFBQUFBQUFBQUFBQUFBQUFCd1pDUndBQlJ3bDNjZ0FBQUFCSlJVNUVya0pnZ2c9PSIvPjxsaW5lYXJHcmFkaWVudCBpZD0iUCIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiLz48bGluZWFyR3JhZGllbnQgaWQ9ImcxIiB4MT0iMjMiIHkxPSI0ODkiIHgyPSI0ODkiIHkyPSIyMyIgaHJlZj0iI1AiPjxzdG9wIHN0b3AtY29sb3I9IiNmY2M2MGUiLz48c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiNlOTJlMjkiLz48L2xpbmVhckdyYWRpZW50PjwvZGVmcz48c3R5bGU+LmF7ZmlsbDp1cmwoI2cxKX08L3N0eWxlPjx1c2UgIGhyZWY9IiNpbWcxIiB4PSI3NSIgeT0iNzYiLz48cGF0aCBmaWxsLXJ1bGU9ImV2ZW5vZGQiIGNsYXNzPSJhIiBkPSJtNTEyIDc4LjR2MzU1LjJjMCA0My4yLTM1LjIgNzguNC03OC40IDc4LjRoLTM1NS4yYy00My4yIDAtNzguNC0zNS4yLTc4LjQtNzguNHYtMzU1LjJjMC00My4yIDM1LjItNzguNCA3OC40LTc4LjRoMzU1LjJjNDMuMiAwIDc4LjQgMzUuMiA3OC40IDc4LjR6bS0zMjQuMyAxNzkuNWMwIDM0LjIgMjcuOSA2MiA2MiA2MmgxMi42YzM0LjEgMCA2Mi0yNy44IDYyLTYydi0xMDEuOWMwLTM0LjItMjcuOS02Mi02Mi02MmgtMTIuNmMtMzQuMSAwLTYyIDI3LjgtNjIgNjJ6bTI0IDB2LTEwMS45YzAtMjEgMTcuMS0zOCAzOC0zOGgxMi42YzIwLjkgMCAzOCAxNyAzOCAzOHYxMDEuOWMwIDIxLTE3LjEgMzgtMzggMzhoLTEyLjZjLTIwLjkgMC0zOC0xNy0zOC0zOHptMTY1LjQtNi4zYzAtNi42LTUuMy0xMi0xMi0xMi02LjYgMC0xMiA1LjQtMTIgMTIgMCA1My42LTQzLjUgOTcuMi05Ny4xIDk3LjItNTMuNiAwLTk3LjEtNDMuNi05Ny4xLTk3LjIgMC02LjYtNS40LTExLjktMTItMTEuOS02LjcgMC0xMiA1LjMtMTIgMTEuOSAwIDYyLjggNDcuOSAxMTQuNSAxMDkuMSAxMjAuNnYzMy44YzAgNi42IDUuNCAxMiAxMiAxMiA2LjYgMCAxMi01LjQgMTItMTJ2LTMzLjhjNjEuMi02LjEgMTA5LjEtNTcuOCAxMDkuMS0xMjAuNnoiLz48L3N2Zz4=',
+ 'recorder','Recorder','Online voice recorder in the browser with cloud storage. Take voice memos by recording through your mic directly in your web browser on any device.',0,0,'https://recorder.puter.com/index.html',1,0,0,'2022-10-21 03:36:06',NULL,NULL,NULL);
diff --git a/packages/backend/src/services/drivers/CoercionService.js b/packages/backend/src/services/drivers/CoercionService.js
new file mode 100644
index 00000000..bd64018b
--- /dev/null
+++ b/packages/backend/src/services/drivers/CoercionService.js
@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("../BaseService");
+const { TypeSpec } = require("./meta/Construct");
+const { TypedValue } = require("./meta/Runtime");
+
+class CoercionService extends BaseService {
+ static MODULES = {
+ axios: require('axios'),
+ }
+
+ async _construct () {
+ this.coercions_ = [];
+ }
+
+ async _init () {
+ this.coercions_.push({
+ produces: {
+ $: 'stream',
+ content_type: 'image'
+ },
+ consumes: {
+ $: 'string:url:web',
+ content_type: 'image'
+ },
+ coerce: async typed_value => {
+ const response = await CoercionService.MODULES.axios.get(typed_value.value, {
+ responseType: 'stream',
+ });
+
+
+ return new TypedValue({
+ $: 'stream',
+ content_type: response.headers['content-type'],
+ }, response.data);
+ }
+ });
+ }
+
+ /**
+ * Attempt to coerce a TypedValue to a target TypeSpec.
+ * Note: this is implemented similarly to MultiValue.get.
+ * @param {*} target - the target TypeSpec
+ * @param {*} typed_value - the TypedValue to coerce
+ * @returns {TypedValue|undefined} - the coerced TypedValue, or undefined
+ */
+ async coerce (target, typed_value) {
+ target = TypeSpec.adapt(target);
+ const target_hash = target.hash();
+
+ const current_type = TypeSpec.adapt(typed_value.type);
+
+ if ( target.equals(current_type) ) {
+ return typed_value;
+ }
+
+ if ( typed_value.calculated_coercions_[target_hash] ) {
+ return typed_value.calculated_coercions_[target_hash];
+ }
+
+ const coercions = this.coercions_.filter(coercion => {
+ const produces = TypeSpec.adapt(coercion.produces);
+ return target.equals(produces);
+ });
+
+ for ( const coercion of coercions ) {
+ const available = await this.coerce(coercion.consumes, typed_value);
+ if ( ! available ) continue;
+ const coerced = await coercion.coerce(available);
+ typed_value.calculated_coercions_[target_hash] = coerced;
+ return coerced;
+ }
+
+ return undefined;
+ }
+}
+
+module.exports = { CoercionService };
diff --git a/packages/backend/src/services/drivers/DriverError.js b/packages/backend/src/services/drivers/DriverError.js
new file mode 100644
index 00000000..61edd3f0
--- /dev/null
+++ b/packages/backend/src/services/drivers/DriverError.js
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class DriverError {
+ static create (source) {
+ return new DriverError({ source });
+ }
+ constructor ({ source, message }) {
+ this.source = source;
+ this.message = source?.message || message;
+ }
+
+ serialize () {
+ return {
+ $: 'heyputer:api/DriverError',
+ message: this.message,
+ };
+ }
+}
+
+module.exports = {
+ DriverError
+};
diff --git a/packages/backend/src/services/drivers/DriverService.js b/packages/backend/src/services/drivers/DriverService.js
new file mode 100644
index 00000000..ce0f8181
--- /dev/null
+++ b/packages/backend/src/services/drivers/DriverService.js
@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { Context } = require("../../util/context");
+const APIError = require("../../api/APIError");
+const { DriverError } = require("./DriverError");
+const { Value, TypedValue } = require("./meta/Runtime");
+const { ServiceImplementation, EntityStoreImplementation } = require("./implementations/EntityStoreImplementation");
+
+/**
+ * DriverService provides the functionality of Puter drivers.
+ */
+class DriverService extends AdvancedBase {
+ static MODULES = {
+ types: require('./types'),
+ }
+
+ constructor ({ services }) {
+ super();
+ this.services = services;
+
+ // TODO: move this to an init method
+ this.log = services.get('log-service').create(this.constructor.name);
+ this.errors = services.get('error-service').create(this.log);
+
+ this.interfaces = require('./interfaces');
+
+ this.interface_to_implementation = {
+ 'helloworld': new (require('./implementations/HelloWorld').HelloWorld)(),
+ 'puter-kvstore': new (require('./implementations/DBKVStore').DBKVStore)(),
+ 'puter-apps': new EntityStoreImplementation({ service: 'es:app' }),
+ 'puter-subdomains': new EntityStoreImplementation({ service: 'es:subdomain' }),
+ };
+ }
+
+ get_interface (interface_name) {
+ return this.interfaces[interface_name];
+ }
+
+ async call (...a) {
+ try {
+ return await this._call(...a);
+ } catch ( e ) {
+ console.error(e);
+ return this._driver_response_from_error(e);
+ }
+ }
+
+ async _call (interface_name, method, args) {
+ const processed_args = await this._process_args(interface_name, method, args);
+ if ( Context.get('test_mode') ) {
+ processed_args.test_mode = true;
+ }
+
+ const actor = Context.get('actor');
+ if ( ! actor ) {
+ throw Error('actor not found in context');
+ }
+
+ const services = Context.get('services');
+ const svc_permission = services.get('permission');
+
+ const perm = await svc_permission.check(actor, `driver:${interface_name}:${method}`);
+ if ( ! perm ) {
+ throw APIError.create('permission_denied');
+ }
+
+ const instance = this.interface_to_implementation[interface_name];
+ if ( ! instance ) {
+ throw APIError.create('no_implementation_available', null, { interface_name })
+ }
+ const meta = await instance.get_response_meta();
+ const sla_override = await this.maybe_get_sla(interface_name, method);
+ try {
+ let result = await instance.call(method, processed_args, sla_override);
+ if ( result instanceof TypedValue ) {
+ const interface_ = this.interfaces[interface_name];
+ let desired_type = interface_.methods[method]
+ .result_choices[0].type;
+ const svc_coercion = services.get('coercion');
+ result = await svc_coercion.coerce(desired_type, result);
+ // meta.type = result.type.toString(),
+ }
+ return { success: true, ...meta, result };
+ } catch ( e ) {
+ let for_user = (e instanceof APIError) || (e instanceof DriverError);
+ if ( ! for_user ) this.errors.report(`driver:${interface_name}:${method}`, {
+ source: e,
+ trace: true,
+ // TODO: alarm will not be suitable for all errors.
+ alarm: true,
+ extra: {
+ args,
+ }
+ });
+ return this._driver_response_from_error(e, meta);
+ }
+ }
+
+ async _driver_response_from_error (e, meta) {
+ let serializable = (e instanceof APIError) || (e instanceof DriverError);
+ if ( serializable ) {
+ console.log('Serialized error test', JSON.stringify(
+ e.serialize(), null, 2
+ ))
+ console.log('Serialized error message: ', e.serialize().message)
+ }
+ return {
+ success: false,
+ ...meta,
+ error: serializable ? e.serialize() : e.message,
+ };
+ }
+
+ async list_interfaces () {
+ return this.interfaces;
+ }
+
+ async maybe_get_sla (interface_name, method) {
+ const services = this.services;
+ const fs = services.get('filesystem');
+
+ return false;
+ }
+
+ async _process_args (interface_name, method_name, args) {
+ // Note: 'interface' is a strict mode reserved word.
+ const interface_ = this.interfaces[interface_name];
+ if ( ! interface_ ) {
+ throw APIError.create('interface_not_found', null, { interface_name });
+ }
+
+ const processed_args = {};
+ const method = interface_.methods[method_name];
+ if ( ! method ) {
+ throw APIError.create('method_not_found', null, { interface_name, method_name });
+ }
+ for ( const [arg_name, arg_descriptor] of Object.entries(method.parameters) ) {
+ const arg_value = args[arg_name];
+ const arg_behaviour = this.modules.types[arg_descriptor.type];
+
+ // TODO: eventually put this in arg behaviour base class.
+ // There's a particular way I want to do this that involves
+ // a trait for extensible behaviour.
+ if ( arg_value === undefined && arg_descriptor.required ) {
+ throw APIError.create('missing_required_argument', null, {
+ interface_name,
+ method_name,
+ arg_name,
+ });
+ }
+
+ const ctx = Context.get();
+
+ try {
+ processed_args[arg_name] = await arg_behaviour.consolidate(
+ ctx, arg_value, { arg_descriptor, arg_name });
+ } catch ( e ) {
+ throw APIError.create('argument_consolidation_failed', null, {
+ interface_name,
+ method_name,
+ arg_name,
+ message: e.message,
+ });
+ }
+ }
+
+ return processed_args;
+ }
+}
+
+module.exports = {
+ DriverService,
+};
diff --git a/packages/backend/src/services/drivers/FileFacade.js b/packages/backend/src/services/drivers/FileFacade.js
new file mode 100644
index 00000000..72bf5d2a
--- /dev/null
+++ b/packages/backend/src/services/drivers/FileFacade.js
@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { Context } = require("../../util/context");
+const { MultiValue } = require("../../util/multivalue");
+const { stream_to_buffer } = require("../../util/streamutil");
+const { PassThrough } = require("stream");
+
+/**
+ * FileFacade
+ *
+ * This class is used to provide a unified interface for
+ * passing files through the Puter Driver API, and avoiding
+ * unnecessary work such as downloading the file from S3
+ * (when a Puter file is specified) in case the underlying
+ * implementation can accept S3 bucket information instead
+ * of the file's contents.
+ *
+ *
+ */
+class FileFacade extends AdvancedBase {
+ static OUT_TYPES = {
+ S3_INFO: { key: 's3-info' },
+ STREAM: { key: 'stream' },
+ }
+
+ static MODULES = {
+ axios: require('axios'),
+ }
+
+ constructor (...a) {
+ super(...a);
+
+ this.values = new MultiValue();
+
+ this.values.add_factory('fs-node', 'uid', async uid => {
+ const context = Context.get();
+ const services = context.get('services');
+ const svc_filesystem = services.get('filesystem');
+ const fsNode = await svc_filesystem.node({ uid });
+ return fsNode;
+ });
+
+ this.values.add_factory('fs-node', 'path', async path => {
+ const context = Context.get();
+ const services = context.get('services');
+ const svc_filesystem = services.get('filesystem');
+ const fsNode = await svc_filesystem.node({ path });
+ return fsNode;
+ });
+
+ this.values.add_factory('s3-info', 'fs-node', async fsNode => {
+ try {
+ return await fsNode.get('s3:location');
+ } catch (e) {
+ return null;
+ }
+ });
+
+ this.values.add_factory('stream', 'fs-node', async fsNode => {
+ if ( ! await fsNode.exists() ) return null;
+
+ const context = Context.get();
+ const services = context.get('services');
+ const svc_filesystem = services.get('filesystem');
+
+ const dst_stream = new PassThrough();
+
+ svc_filesystem.read(context, dst_stream, {
+ fsNode,
+ user: context.get('user'),
+ });
+
+ return dst_stream;
+ });
+
+ this.values.add_factory('stream', 'web_url', async web_url => {
+ const response = await FileFacade.MODULES.axios.get(web_url, {
+ responseType: 'stream',
+ });
+
+ return response.data;
+ });
+
+ this.values.add_factory('stream', 'data_url', async data_url => {
+ const data = data_url.split(',')[1];
+ const buffer = Buffer.from(data, 'base64');
+ const stream = new PassThrough();
+ stream.end(buffer);
+ return stream;
+ });
+
+ this.values.add_factory('buffer', 'stream', async stream => {
+ return await stream_to_buffer(stream);
+ });
+ }
+
+ set (k, v) { this.values.set(k, v); }
+ get (k) { return this.values.get(k); }
+
+
+}
+
+module.exports = {
+ FileFacade,
+};
diff --git a/packages/backend/src/services/drivers/implementations/BaseImplementation.js b/packages/backend/src/services/drivers/implementations/BaseImplementation.js
new file mode 100644
index 00000000..8d0ce505
--- /dev/null
+++ b/packages/backend/src/services/drivers/implementations/BaseImplementation.js
@@ -0,0 +1,189 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { Context } = require("../../../util/context");
+const APIError = require("../../../api/APIError");
+const { AppUnderUserActorType, Actor, UserActorType } = require("../../auth/Actor");
+const { BaseOperation } = require("../../OperationTraceService");
+const { CodeUtil } = require("../../../codex/CodeUtil");
+
+/**
+ * Base class for all driver implementations.
+ */
+class BaseImplementation extends AdvancedBase {
+ constructor (...a) {
+ super(...a);
+ const methods = this._get_merged_static_object('METHODS');
+ // Turn each method into an operation
+ for ( const k in methods ) {
+ methods[k] = CodeUtil.mrwrap(methods[k], BaseOperation, {
+ name: `${this.constructor.ID}:${k}`,
+ });
+ };
+ this.methods = methods;
+ this.sla = this._get_merged_static_object('SLA');
+ }
+
+ async call (method, args) {
+ if ( ! this.methods[method] ) {
+ throw new Error(`method not found: ${method}`);
+ }
+
+ const pseudo_this = Object.assign({}, this);
+
+ const context = Context.get();
+ pseudo_this.context = context;
+ pseudo_this.services = context.get('services');
+ const services = context.get('services');
+ pseudo_this.log = services.get('log-service').create(this.constructor.name);
+
+ await this._sla_enforcement(method);
+
+ return await this.methods[method].call(pseudo_this, args);
+ }
+
+ async _sla_enforcement (method) {
+ const context = Context.get();
+ const method_key = `${this.constructor.ID}:${method}`;
+ const svc_sla = services.get('sla');
+
+ // System SLA enforcement
+ {
+ const sla_key = `driver:impl:${method_key}`;
+ const sla = await svc_sla.get('system', sla_key);
+
+ const sys_method_key = `system:${method_key}`;
+
+ // short-term rate limiting
+ if ( sla?.rate_limit ) {
+ const svc_rateLimit = services.get('rate-limit');
+ let eventual_success = false;
+ for ( let i = 0 ; i < 60 ; i++ ) {
+ try {
+ await svc_rateLimit.check_and_increment(sys_method_key, sla.rate_limit.max, sla.rate_limit.period);
+ eventual_success = true;
+ break;
+ } catch ( e ) {
+ if (
+ ! ( e instanceof APIError ) ||
+ e.fields.code !== 'rate_limit_exceeded'
+ ) throw e;
+ await new Promise((resolve) => setTimeout(resolve, 1000));
+ }
+ }
+ if ( ! eventual_success ) {
+ throw APIError.create('server_rate_exceeded');
+ }
+ }
+ }
+
+ // test_mode is checked to prevent rate limiting when it is enabled
+ const test_mode = context.get('test_mode');
+
+ // User SLA enforcement
+ {
+ const actor = context.get('actor').get_related_actor(UserActorType);
+
+ const user_is_verified = !! actor.type.user.email_confirmed;
+
+ const sla_key = `driver:impl:${method_key}`;
+ const sla = await svc_sla.get(
+ user_is_verified ? 'user_verified' : 'user_unverified',
+ sla_key
+ );
+
+ console.log('SLA KEY', sla_key, 'USER KEY', user_is_verified ? 'user_verified' : 'user_unverified');
+
+ const user_method_key = `actor:${actor.uid}:${method_key}`;
+
+ // short-term rate limiting
+ if ( sla?.rate_limit ) {
+ const svc_rateLimit = services.get('rate-limit');
+ await svc_rateLimit.check_and_increment(method_key, sla.rate_limit.max, sla.rate_limit.period);
+ }
+
+ // long-term rate limiting
+ if ( sla?.monthly_limit && ! test_mode ) {
+ const svc_monthlyUsage = services.get('monthly-usage');
+ const count = await svc_monthlyUsage.check(
+ actor, {
+ 'driver.interface': this.constructor.INTERFACE,
+ 'driver.implementation': this.constructor.ID,
+ 'driver.method': method,
+ });
+ if ( count >= sla.monthly_limit ) {
+ throw APIError.create('monthly_limit_exceeded', null, {
+ method_key,
+ limit: sla.monthly_limit,
+ });
+ }
+ }
+ }
+
+ // App SLA enforcement
+ await (async () => {
+ const actor = context.get('actor');
+ if ( ! ( actor.type instanceof AppUnderUserActorType ) ) return;
+
+ const sla_key = `driver:impl:${method_key}`;
+ const sla = await svc_sla.get('app_default', sla_key);
+
+ // long-term rate limiting
+ if ( sla?.monthly_limit && ! test_mode ) {
+ const svc_monthlyUsage = services.get('monthly-usage');
+ const count = await svc_monthlyUsage.check(
+ actor, {
+ 'driver.interface': this.constructor.INTERFACE,
+ 'driver.implementation': this.constructor.ID,
+ 'driver.method': method,
+ });
+ if ( count >= sla.monthly_limit ) {
+ throw APIError.create('monthly_limit_exceeded', null, {
+ method_key,
+ limit: sla.monthly_limit,
+ });
+ }
+ }
+ })();
+
+ // Record monthly usage
+ if ( ! test_mode ) {
+ const actor = context.get('actor');
+ const svc_monthlyUsage = services.get('monthly-usage');
+ const extra = {
+ 'driver.interface': this.constructor.INTERFACE,
+ 'driver.implementation': this.constructor.ID,
+ 'driver.method': method,
+ };
+ await svc_monthlyUsage.increment(actor, method_key, extra);
+ }
+ }
+
+ async get_response_meta () {
+ return {
+ driver: this.constructor.ID,
+ driver_version: this.constructor.VERSION,
+ driver_interface: this.constructor.INTERFACE,
+ };
+ }
+}
+
+module.exports = {
+ BaseImplementation,
+};
diff --git a/packages/backend/src/services/drivers/implementations/DBKVStore.js b/packages/backend/src/services/drivers/implementations/DBKVStore.js
new file mode 100644
index 00000000..c203184e
--- /dev/null
+++ b/packages/backend/src/services/drivers/implementations/DBKVStore.js
@@ -0,0 +1,173 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AppUnderUserActorType } = require("../../auth/Actor");
+const { BaseImplementation } = require("./BaseImplementation");
+
+const config = require("../../../config");
+const APIError = require("../../../api/APIError");
+const { DB_READ, DB_WRITE } = require("../../database/consts");
+
+class DBKVStore extends BaseImplementation {
+ static ID = 'public-db-kvstore';
+ static VERSION = '0.0.0';
+ static INTERFACE = 'puter-kvstore';
+ static MODULES = {
+ murmurhash: require('murmurhash'),
+ }
+ static METHODS = {
+ get: async function ({ key }) {
+ const actor = this.context.get('actor');
+
+ // If the actor is an app then it gets its own KV store.
+ // The way this is implemented isn't ideal for future behaviour;
+ // a KV implementation specified by the user would have parameters
+ // that are scoped to the app, so this should eventually be
+ // changed to get the app ID from the same interface that would
+ // be used to obtain per-app user-specified implementation params.
+ const app = actor.type?.app ?? undefined;
+ const user = actor.type?.user ?? undefined;
+
+ if ( ! user ) throw new Error('User not found');
+
+ const db = this.services.get('database').get(DB_READ, 'kvstore');
+ const key_hash = this.modules.murmurhash.v3(key);
+ const kv = app ? await db.read(
+ `SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1`,
+ [ user.id, app.uid, key_hash ]
+ ) : await db.read(
+ `SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global') AND kkey_hash=? LIMIT 1`,
+ [ user.id, key_hash ]
+ );
+
+ return kv[0] ? { key: kv[0].key, value: kv[0].value } : null;
+ },
+ set: async function ({ key, value }) {
+ const actor = this.context.get('actor');
+
+ // Validate the key
+ // get() doesn't String() the key but it only passes it to
+ // murmurhash.v3() so it doesn't need to ¯\_(ツ)_/¯
+ key = String(key);
+ if ( Buffer.byteLength(key, 'utf8') > config.kv_max_key_size ) {
+ throw new Error(`key is too large. Max size is ${config.kv_max_key_size}.`);
+ }
+
+ // Validate the value
+ value = value === undefined ? null : String(value);
+ if (
+ value !== null &&
+ Buffer.byteLength(value, 'utf8') > config.kv_max_value_size
+ ) {
+ throw new Error(`value is too large. Max size is ${config.kv_max_value_size}.`);
+ }
+
+ const app = actor.type?.app ?? undefined;
+ const user = actor.type?.user ?? undefined;
+ if ( ! user ) throw new Error('User not found');
+
+ const db = this.services.get('database').get(DB_WRITE, 'kvstore');
+ const key_hash = this.modules.murmurhash.v3(key);
+
+ await db.write(
+ `INSERT INTO kv
+ (user_id, app, kkey_hash, kkey, value)
+ VALUES
+ (?, ?, ?, ?, ?)
+ ON DUPLICATE KEY UPDATE
+ value = ?`,
+ [ user.id, app?.uid ?? 'global', key_hash, key, value, value ]
+ );
+
+ return true;
+ },
+ del: async function ({ key }) {
+ const actor = this.context.get('actor');
+
+ const app = actor.type?.app ?? undefined;
+ const user = actor.type?.user ?? undefined;
+ if ( ! user ) throw new Error('User not found');
+
+ const db = this.services.get('database').get(DB_WRITE, 'kvstore');
+ const key_hash = this.modules.murmurhash.v3(key);
+
+ await db.write(
+ `DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?`,
+ [ user.id, app?.uid ?? 'global', key_hash ]
+ );
+
+ return true;
+ },
+ list: async function ({ as }) {
+ const actor = this.context.get('actor');
+
+ const app = actor.type?.app ?? undefined;
+ const user = actor.type?.user ?? undefined;
+
+ if ( ! user ) throw new Error('User not found');
+
+ const db = this.services.get('database').get(DB_READ, 'kvstore');
+ let rows = app ? await db.read(
+ `SELECT kkey, value FROM kv WHERE user_id=? AND app=?`,
+ [ user.id, app.uid ]
+ ) : await db.read(
+ `SELECT kkey, value FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global')`,
+ [ user.id ]
+ );
+
+ rows = rows.map(row => ({
+ key: row.kkey,
+ value: row.value,
+ }));
+
+ as = as || 'entries';
+
+ if ( ! ['keys','values','entries'].includes(as) ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'as',
+ expected: '"keys", "values", or "entries"',
+ });
+ }
+
+ if ( as === 'keys' ) rows = rows.map(row => row.key);
+ else if ( as === 'values' ) rows = rows.map(row => row.value);
+
+ return rows;
+ },
+ flush: async function () {
+ const actor = this.context.get('actor');
+
+ const app = actor.type?.app ?? undefined;
+ const user = actor.type?.user ?? undefined;
+ if ( ! user ) throw new Error('User not found');
+
+ const db = this.services.get('database').get(DB_WRITE, 'kvstore');
+
+ await db.write(
+ `DELETE FROM kv WHERE user_id=? AND app=?`,
+ [ user.id, app?.uid ?? 'global' ]
+ );
+
+ return true;
+ }
+ }
+}
+
+module.exports = {
+ DBKVStore,
+}
diff --git a/packages/backend/src/services/drivers/implementations/EntityStoreImplementation.js b/packages/backend/src/services/drivers/implementations/EntityStoreImplementation.js
new file mode 100644
index 00000000..6e98f9b9
--- /dev/null
+++ b/packages/backend/src/services/drivers/implementations/EntityStoreImplementation.js
@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const APIError = require("../../../api/APIError");
+const { Entity } = require("../../../om/entitystorage/Entity");
+const { Or, And, Eq } = require("../../../om/query/query");
+const { BaseImplementation } = require("./BaseImplementation");
+
+const _fetch_based_on_complex_id = async (self, id) => {
+ // Ensure `id` is an object and get its keys
+ if ( ! id || typeof id !== 'object' || Array.isArray(id) ) {
+ throw APIError.create('invalid_id', null, { id });
+ }
+
+ const id_keys = Object.keys(id);
+ // sort keys alphabetically
+ id_keys.sort();
+
+ // Ensure key set is valid based on redundant keys listing
+ const svc_es = self.services.get(self.service);
+ const redundant_identifiers = svc_es.om.redundant_identifiers ?? [];
+
+ let match_found = false;
+ for ( let key of redundant_identifiers ) {
+ // Either a single key or a list
+ key = Array.isArray(key) ? key : [key];
+
+ // All keys in the list must be present in the id
+ for ( let i=0 ; i < key.length ; i++ ) {
+ if ( ! id_keys.includes(key[i]) ) {
+ break;
+ }
+ if ( i === key.length - 1 ) {
+ match_found = true;
+ break;
+ }
+ }
+ }
+
+ if ( ! match_found ) {
+ throw APIError.create('invalid_id', null, { id });
+ }
+
+ // Construct a query predicate based on the keys
+ const key_eqs = [];
+ for ( const key of id_keys ) {
+ key_eqs.push(new Eq({
+ key,
+ value: id[key],
+ }));
+ }
+ let predicate = new And({ children: key_eqs });
+
+ // Perform a select
+ const entities = await svc_es.select({ predicate });
+ if ( entities.length === 0 ) {
+ return null;
+ }
+
+ console.log('WHAT ISAGERSGAREWHGwr', entities)
+
+ // Ensure there is only one result
+ return entities[0];
+}
+
+const _fetch_based_on_either_id = async (self, uid, id) => {
+ if ( uid ) {
+ const svc_es = self.services.get(self.service);
+ return await svc_es.read(uid);
+ }
+
+ return await _fetch_based_on_complex_id(self, id);
+}
+
+class EntityStoreImplementation extends BaseImplementation {
+ constructor ({ service }) {
+ super();
+ this.service = service;
+ }
+ static METHODS = {
+ create: async function ({ object }) {
+ const svc_es = this.services.get(this.service);
+ if ( object.hasOwnProperty(svc_es.om.primary_identifier) ) {
+ throw APIError.create('field_not_allowed_for_create', null, { key: svc_es.om.primary_identifier });
+ }
+ const entity = await Entity.create({ om: svc_es.om }, object);
+ return await svc_es.create(entity);
+ },
+ update: async function ({ object, id }) {
+ const svc_es = this.services.get(this.service);
+ // if ( ! object.hasOwnProperty(svc_es.om.primary_identifier) ) {
+ // throw APIError.create('field_required_for_update', null, { key: svc_es.om.primary_identifier });
+ // }
+ const entity = await Entity.create({ om: svc_es.om }, object);
+ return await svc_es.update(entity, id);
+ },
+ upsert: async function ({ object, id }) {
+ const svc_es = this.services.get(this.service);
+ const entity = await Entity.create({ om: svc_es.om }, object);
+ return await svc_es.upsert(entity, id);
+ },
+ read: async function ({ uid, id }) {
+ if ( ! uid && ! id ) {
+ throw APIError.create('xor_field_missing', null, {
+ names: ['uid', 'id'],
+ });
+ }
+
+ const entity = await _fetch_based_on_either_id(this, uid, id);
+ if ( ! entity ) {
+ throw APIError.create('entity_not_found', null, {
+ identifier: uid
+ });
+ }
+ return await entity.get_client_safe();
+ },
+ select: async function (options) {
+ const svc_es = this.services.get(this.service);
+ const entities = await svc_es.select(options);
+ const client_safe_entities = [];
+ for ( const entity of entities ) {
+ client_safe_entities.push(await entity.get_client_safe());
+ }
+ return client_safe_entities;
+ },
+ delete: async function ({ uid, id }) {
+ if ( ! uid && ! id ) {
+ throw APIError.create('xor_field_missing', null, {
+ names: ['uid', 'id'],
+ });
+ }
+
+ if ( id && ! uid ) {
+ const entity = await _fetch_based_on_complex_id(this, id);
+ if ( ! entity ) {
+ throw APIError.create('entity_not_found', null, {
+ identifier: id
+ });
+ }
+ const svc_es = this.services.get(this.service);
+ uid = await entity.get(svc_es.om.primary_identifier);
+ }
+
+ const svc_es = this.services.get(this.service);
+ return await svc_es.delete(uid);
+ },
+ };
+}
+
+module.exports = {
+ EntityStoreImplementation,
+};
diff --git a/packages/backend/src/services/drivers/implementations/HelloWorld.js b/packages/backend/src/services/drivers/implementations/HelloWorld.js
new file mode 100644
index 00000000..7513e691
--- /dev/null
+++ b/packages/backend/src/services/drivers/implementations/HelloWorld.js
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Context } = require("../../../util/context");
+const { BaseImplementation } = require("./BaseImplementation");
+
+class HelloWorld extends BaseImplementation {
+ static ID = 'public-helloworld';
+ static VERSION = '0.0.0';
+ static INTERFACE = 'helloworld';
+ static SLA = {
+ greet: {
+ rate_limit: {
+ max: 10,
+ period: 30000,
+ },
+ monthly_limit: Math.pow(1, 6),
+ },
+ }
+ static METHODS = {
+ greet: async function ({ subject }) {
+ return `Hello, ${subject ?? 'World'}!`
+ }
+ }
+}
+
+module.exports = {
+ HelloWorld,
+};
diff --git a/packages/backend/src/services/drivers/implementations/PuterDriverProxy.js b/packages/backend/src/services/drivers/implementations/PuterDriverProxy.js
new file mode 100644
index 00000000..c34495a5
--- /dev/null
+++ b/packages/backend/src/services/drivers/implementations/PuterDriverProxy.js
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+const ENDPOINT = 'https://api.puter.com/drivers/call';
+
+/*
+
+Fetch example:
+
+await fetch("https://api.puter.local/drivers/call", {
+ "headers": {
+ "Content-Type": "application/json",
+ "Authorization": "Bearer ",
+ },
+ "body": JSON.stringify({ interface: '...', method: '...', args: { ... } }),
+ "method": "POST",
+});
+
+*/
+
+class PuterDriverProxy extends AdvancedBase {
+ static MODULES = {
+ axios: require('axios'),
+ }
+
+ constructor ({ target }) {
+ this.target = target;
+ }
+
+ async call (method, args) {
+ const require = this.require;
+ const axios = require('axios');
+
+ // TODO: We need the BYOK feature before we can implement this
+ }
+}
+
+module.exports = PuterDriverProxy;
diff --git a/packages/backend/src/services/drivers/interfaces.js b/packages/backend/src/services/drivers/interfaces.js
new file mode 100644
index 00000000..3225c4aa
--- /dev/null
+++ b/packages/backend/src/services/drivers/interfaces.js
@@ -0,0 +1,274 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const ENTITY_STORAGE_INTERFACE = {
+ methods: {
+ create: {
+ parameters: {
+ object: {
+ type: 'json',
+ subtype: 'object',
+ required: true,
+ },
+ }
+ },
+ read: {
+ parameters: {
+ uid: { type: 'string' },
+ id: { type: 'json' },
+ }
+ },
+ select: {
+ parameters: {
+ predicate: { type: 'json' },
+ offset: { type: 'number' },
+ limit: { type: 'number' },
+ }
+ },
+ update: {
+ parameters: {
+ id: { type: 'json' },
+ object: {
+ type: 'json',
+ subtype: 'object',
+ required: true,
+ },
+ }
+ },
+ upsert: {
+ parameters: {
+ id: { type: 'json' },
+ object: {
+ type: 'json',
+ subtype: 'object',
+ required: true,
+ },
+ }
+ },
+ delete: {
+ parameters: {
+ uid: { type: 'string' },
+ id: { type: 'json' },
+ }
+ },
+ },
+}
+
+module.exports = {
+ 'helloworld': {
+ description: 'A simple driver that returns a greeting.',
+ methods: {
+ greet: {
+ description: 'Returns a greeting.',
+ parameters: {
+ subject: {
+ type: 'string',
+ optional: true,
+ },
+ },
+ result: { type: 'string' },
+ }
+ }
+ },
+ // Note: these are all prefixed with 'puter-' to avoid name collisions
+ // with possible future support for user-contributed driver interfaces.
+ 'puter-ocr': {
+ description: 'Optical character recognition.',
+ methods: {
+ recognize: {
+ description: 'Recognize text in an image or document.',
+ parameters: {
+ source: {
+ type: 'file',
+ },
+ },
+ result: { type: 'image' },
+ },
+ },
+ },
+ 'puter-kvstore': {
+ description: 'A simple key-value store.',
+ methods: {
+ get: {
+ description: 'Get a value by key.',
+ parameters: { key: { type: 'string', required: true } },
+ result: { type: 'json' },
+ },
+ set: {
+ description: 'Set a value by key.',
+ parameters: {
+ key: { type: 'string', required: true, },
+ value: { type: 'json' }
+ },
+ result: { type: 'void' },
+ },
+ del: {
+ description: 'Delete a value by key.',
+ parameters: { key: { type: 'string' } },
+ result: { type: 'void' },
+ },
+ list: {
+ description: 'List all key-value pairs.',
+ parameters: {
+ as: {
+ type: 'string',
+ },
+ },
+ result: { type: 'array' },
+ },
+ flush: {
+ description: 'Delete all key-value pairs.',
+ parameters: {},
+ result: { type: 'void' },
+ },
+ incr: {
+ description: 'Increment a value by key.',
+ parameters: {
+ key: { type: 'string', required: true, },
+ amount: { type: 'number' },
+ },
+ result: { type: 'number' },
+ },
+ decr: {
+ description: 'Increment a value by key.',
+ parameters: {
+ key: { type: 'string', required: true, },
+ amount: { type: 'number' },
+ },
+ result: { type: 'number' },
+ },
+ /*
+ expireat: {
+ description: 'Set a key\'s time-to-live.',
+ parameters: {
+ key: { type: 'string', required: true, },
+ timestamp: { type: 'number', required: true, },
+ },
+ },
+ expire: {
+ description: 'Set a key\'s time-to-live.',
+ parameters: {
+ key: { type: 'string', required: true, },
+ ttl: { type: 'number', required: true, },
+ },
+ }
+ */
+ }
+ },
+ 'puter-chat-completion': {
+ description: 'Chatbot.',
+ methods: {
+ complete: {
+ description: 'Get completions for a chat log.',
+ parameters: {
+ messages: { type: 'json' },
+ vision: { type: 'flag' },
+ },
+ result: { type: 'json' }
+ }
+ }
+ },
+ 'puter-image-generation': {
+ description: 'AI Image Generation.',
+ methods: {
+ generate: {
+ description: 'Generate an image from a prompt.',
+ parameters: {
+ prompt: { type: 'string' },
+ },
+ result_choices: [
+ {
+ names: ['image'],
+ type: {
+ $: 'stream',
+ content_type: 'image',
+ }
+ },
+ {
+ names: ['url'],
+ type: {
+ $: 'string:url:web',
+ content_type: 'image',
+ }
+ },
+ ],
+ result: {
+ description: 'URL of the generated image.',
+ type: 'string'
+ }
+ }
+ }
+ },
+ 'puter-tts': {
+ description: 'Text-to-speech.',
+ methods: {
+ list_voices: {
+ description: 'List available voices.',
+ parameters: {},
+ },
+ synthesize: {
+ description: 'Synthesize speech from text.',
+ parameters: {
+ text: { type: 'string' },
+ voice: { type: 'string' },
+ language: { type: 'string' },
+ ssml: { type: 'flag' },
+ },
+ result_choices: [
+ {
+ names: ['audio'],
+ type: {
+ $: 'stream',
+ content_type: 'audio',
+ }
+ },
+ ]
+ },
+ }
+ },
+ 'puter-analytics': {
+ no_sdk: true,
+ description: 'Analytics.',
+ methods: {
+ create_trace: {
+ description: 'Get a trace UID.',
+ parameters: {
+ trace_id: { type: 'string', optional: true },
+ },
+ result: { type: 'string' }
+ },
+ record: {
+ description: 'Record an event.',
+ parameters: {
+ trace_id: { type: 'string', optional: true },
+ tags: { type: 'json' },
+ fields: { type: 'json' },
+ },
+ result: { type: 'void' }
+ }
+ }
+ },
+ 'puter-apps': {
+ ...ENTITY_STORAGE_INTERFACE,
+ description: 'Manage a developer\'s apps on Puter.',
+ },
+ 'puter-subdomains': {
+ ...ENTITY_STORAGE_INTERFACE,
+ description: 'Manage subdomains on Puter.',
+ }
+};
diff --git a/packages/backend/src/services/drivers/meta/Construct.js b/packages/backend/src/services/drivers/meta/Construct.js
new file mode 100644
index 00000000..feaa9ed8
--- /dev/null
+++ b/packages/backend/src/services/drivers/meta/Construct.js
@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { BasicBase } = require("puter-js-common/src/bases/BasicBase");
+const types = require("../types");
+const { hash_serializable_object, stringify_serializable_object } = require("../../../util/datautil");
+
+class Construct extends BasicBase {
+ constructor (json, { name } = {}) {
+ super();
+ this.name = name;
+ this.raw = json;
+ this.__process();
+ }
+
+ __process () {
+ if ( this._process ) this._process(this.raw);
+ }
+
+ serialize () {
+ const props = this._get_merged_static_object('PROPERTIES');
+ const serialized = {};
+ for ( const prop_name in props ) {
+ const prop = props[prop_name];
+
+ if ( prop.type === 'object' ) {
+ serialized[prop_name] = this[prop_name]?.serialize?.() ?? null;
+ } else if ( prop.type === 'map' ) {
+ serialized[prop_name] = {};
+ for ( const key in this[prop_name] ) {
+ const object = this[prop_name][key];
+ serialized[prop_name][key] = object.serialize();
+ }
+ } else {
+ serialized[prop_name] = this[prop_name];
+ }
+ }
+ return serialized;
+ }
+}
+
+class Parameter extends Construct {
+ static PROPERTIES = {
+ type: { type: 'object' },
+ optional: { type: 'boolean' },
+ description: { type: 'string' },
+ };
+
+ _process (raw) {
+ this.type = types[raw.type];
+ }
+}
+
+class Method extends Construct {
+ static PROPERTIES = {
+ description: { type: 'string' },
+ parameters: { type: 'map' },
+ result: { type: 'object' },
+ };
+
+ _process (raw) {
+ this.description = raw.description;
+ this.parameters = {};
+
+ for ( const parameter_name in raw.parameters ) {
+ const parameter = raw.parameters[parameter_name];
+ this.parameters[parameter_name] = new Parameter(
+ parameter, { name: parameter_name });
+ }
+
+ if ( raw.result ) {
+ this.result = new Parameter(raw.result, { name: 'result' });
+ }
+ }
+}
+
+class Interface extends Construct {
+ static PROPERTIES = {
+ description: { type: 'string' },
+ methods: { type: 'map' },
+ };
+
+ _process (raw) {
+ this.description = raw.description;
+ this.methods = {};
+
+ for ( const method_name in raw.methods ) {
+ const method = raw.methods[method_name];
+ this.methods[method_name] = new Method(
+ method, { name: method_name });
+ }
+ }
+}
+
+class TypeSpec extends BasicBase {
+ static adapt (raw) {
+ if ( raw instanceof TypeSpec ) return raw;
+ return new TypeSpec(raw);
+ }
+ constructor (raw) {
+ super();
+ this.raw = raw;
+ }
+
+ equals (other) {
+ return this.raw.$ === other.raw.$;
+ // for ( k in this.raw ) {
+ // if ( this.raw[k] !== other.raw[k] ) return false;
+ // }
+ return true;
+ }
+
+ toString () {
+ return stringify_serializable_object(this.raw);
+ }
+
+ hash () {
+ return hash_serializable_object(this.raw);
+ }
+}
+
+// NEXT: class Type extends Construct
+
+module.exports = {
+ Construct,
+ Parameter,
+ Method,
+ Interface,
+ TypeSpec,
+}
diff --git a/packages/backend/src/services/drivers/meta/Runtime.js b/packages/backend/src/services/drivers/meta/Runtime.js
new file mode 100644
index 00000000..c0547802
--- /dev/null
+++ b/packages/backend/src/services/drivers/meta/Runtime.js
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { BasicBase } = require("puter-js-common/src/bases/BasicBase");
+const { TypeSpec } = require("./Construct");
+
+class RuntimeEntity extends BasicBase {
+}
+
+class TypedValue extends RuntimeEntity {
+ constructor (type, value) {
+ super();
+ this.type = TypeSpec.adapt(type);
+ this.value = value;
+ this.calculated_coercions_ = {};
+ }
+}
+
+module.exports = {
+ TypedValue
+};
diff --git a/packages/backend/src/services/drivers/types.js b/packages/backend/src/services/drivers/types.js
new file mode 100644
index 00000000..378a237e
--- /dev/null
+++ b/packages/backend/src/services/drivers/types.js
@@ -0,0 +1,183 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const { is_valid_path } = require("../../filesystem/validation");
+const { is_valid_url, is_valid_uuid4 } = require("../../helpers");
+const { FileFacade } = require("./FileFacade");
+const APIError = require("../../api/APIError");
+
+class BaseType extends AdvancedBase {}
+
+class String extends BaseType {
+ async consolidate (ctx, input) {
+ // undefined means the optional parameter was not provided,
+ // which is different from an empty string.
+ return (
+ input === undefined ||
+ input === null
+ ) ? undefined : '' + input;
+ }
+
+ serialize () { return 'string'; }
+}
+
+class Flag extends BaseType {
+ async consolidate (ctx, input) {
+ return !! input;
+ }
+
+ serialize () { return 'flag'; }
+}
+
+class NumberType extends BaseType {
+ async consolidate (ctx, input, { arg_name, arg_descriptor }) {
+ // Case for optional values
+ if ( input === undefined ) return undefined;
+
+ if ( typeof input !== 'number' ) {
+ throw APIError.create('field_invalid', null, {
+ key: arg_name,
+ expected: 'number',
+ });
+ }
+
+ if ( arg_descriptor.unsigned && input < 0 ) {
+ throw APIError.create('field_invalid', null, {
+ key: arg_name,
+ expected: 'unsigned number',
+ });
+ }
+
+ return input;
+ }
+
+ serialize () { return 'number'; }
+}
+
+class URL extends BaseType {
+ async consolidate (ctx, input, { arg_name }) {
+ if ( ! is_valid_url(input) ) {
+ throw APIError.create('field_invalid', null, {
+ key: arg_name,
+ expected: 'URL',
+ });
+ }
+ return input;
+ }
+
+ serialize () { return 'url'; }
+}
+
+class File extends BaseType {
+ static DOC_INPUT_FORMATS = [
+ 'A puter filepath, like /home/user/file.txt',
+ 'A puter filesystem UUID, like 12345678-1234-1234-1234-123456789abc',
+ 'A URL, like https://example.com/file.txt',
+ 'A base64-encoded string, like data:image/png;base64,iVBORw0K...',
+ ]
+ static DOC_INTERNAL_TYPE = 'An instance of FileFacade'
+
+ static MODULES = {
+ _path: require('path'),
+ }
+
+ async consolidate (ctx, input, { arg_name }) {
+ const result = new FileFacade();
+ // DRY: Part of this is duplicating FSNodeParam, but FSNodeParam is
+ // subject to change in PR #647, so this should be updated later.
+
+ if ( ! ['/','.','~'].includes(input[0]) ) {
+ if ( is_valid_uuid4(input) ) {
+ result.set('uid', input);
+ return result;
+ }
+
+ if ( is_valid_url(input) ) {
+ if ( input.startsWith('data:') ) {
+ result.set('data_url', input);
+ return result;
+ }
+ result.set('web_url', input);
+ return result;
+ }
+
+ }
+
+ if ( input.startsWith('~') ) {
+ const user = ctx.get('user');
+ if ( ! user ) {
+ throw new Error('Cannot use ~ without a user');
+ }
+ const homedir = `/${user.username}`;
+ input = homedir + input.slice(1);
+ }
+
+ if ( ! is_valid_path(input) ) {
+ throw APIError.create('field_invalid', null, {
+ key: arg_name,
+ expected: 'unix-style path or UUID',
+ });
+ }
+
+ result.set('path', this.modules._path.resolve('/', input));
+ return result;
+ }
+
+ serialize () { return 'file'; }
+}
+
+class JSONType extends BaseType {
+ async consolidate (ctx, input, { arg_descriptor, arg_name }) {
+ if ( input === undefined ) return undefined;
+
+ if ( arg_descriptor.subtype ) {
+ const input_json_type =
+ Array.isArray(input) ? 'array' :
+ input === null ? 'null' :
+ typeof input;
+
+ if ( input_json_type === 'null' || input_json_type === 'undefined' ) {
+ return input;
+ }
+
+ if ( input_json_type !== arg_descriptor.subtype ) {
+ throw APIError.create('field_invalid', null, {
+ key: arg_name,
+ expected: `JSON value of type ${arg_descriptor.subtype}`,
+ got: `JSON value of type ${input_json_type}`,
+ });
+ }
+ }
+ return input;
+ }
+
+ serialize () { return 'json'; }
+}
+
+// class WebURLString extends BaseType {
+// }
+
+module.exports = {
+ file: new File(),
+ string: new String(),
+ flag: new Flag(),
+ json: new JSONType(),
+ number: new NumberType(),
+ // 'string:url:web': WebURLString,
+};
diff --git a/packages/backend/src/services/file-cache/FileCacheService.js b/packages/backend/src/services/file-cache/FileCacheService.js
new file mode 100644
index 00000000..3725e7d6
--- /dev/null
+++ b/packages/backend/src/services/file-cache/FileCacheService.js
@@ -0,0 +1,298 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const TeePromise = require("@heyputer/multest/src/util/TeePromise");
+const { AdvancedBase } = require("puter-js-common");
+const { FileTracker } = require("./FileTracker");
+const { pausing_tee } = require("../../util/streamutil");
+
+/**
+ * FileCacheService
+ *
+ * Initial naive cache implementation which stores whole files on disk.
+ * It is assumed that files are only accessed by one server at a given time,
+ * so this will need to be revised when ACL and sharing is implemented.
+ */
+class FileCacheService extends AdvancedBase {
+ static MODULES = {
+ fs: require('fs'),
+ path_: require('path'),
+ }
+
+ constructor ({ services, my_config }) {
+ super({ services });
+
+ this.log = services.get('log-service').create(this.constructor.name);
+ this.errors = services.get('error-service').create(this.log);
+
+ this.disk_limit = my_config.disk_limit;
+ this.disk_max_size = my_config.disk_max_size;
+ this.precache_size = my_config.precache_size;
+ this.path = my_config.path;
+
+ this.ttl = my_config.ttl || (5 * 1000);
+
+ this.precache = new Map();
+ this.uid_to_tracker = new Map();
+
+ this.init();
+
+ this._register_commands(services.get('commands'));
+ }
+
+ get _precache_used () {
+ let used = 0;
+
+ // Iterate over file trackers in PHASE_PRECACHE
+ for (const tracker of this.uid_to_tracker.values()) {
+ if (tracker.phase !== FileTracker.PHASE_PRECACHE) continue;
+ used += tracker.size;
+ }
+
+ return used;
+ }
+
+ get _disk_used () {
+ let used = 0;
+
+ // Iterate over file trackers in PHASE_DISK
+ for (const tracker of this.uid_to_tracker.values()) {
+ if (tracker.phase !== FileTracker.PHASE_DISK) continue;
+ used += tracker.size;
+ }
+
+ return used;
+ }
+
+ async init () {
+ const { fs } = this.modules;
+ // Ensure storage path exists
+ await fs.promises.mkdir(this.path, { recursive: true });
+ }
+
+ _get_path (uid) {
+ const { path_, fs } = this.modules;
+ return path_.join(this.path, uid);
+ }
+
+ async try_get (fsNode, opt_log) {
+ const tracker = this.uid_to_tracker.get(await fsNode.get('uid'));
+
+ if ( ! tracker ) {
+ return null;
+ }
+
+ if ( tracker.age > this.ttl ) {
+ await this.invalidate(fsNode);
+ return null;
+ }
+
+ tracker.touch();
+
+ if ( tracker.phase === FileTracker.PHASE_PRECACHE ) {
+ if ( opt_log ) opt_log.info('obtained from precache');
+ return this.precache.get(await fsNode.get('uid'));
+ }
+
+ if ( tracker.phase === FileTracker.PHASE_DISK ) {
+ if ( opt_log ) opt_log.info('obtained from disk');
+
+ const { fs } = this.modules;
+ const path = this._get_path(await fsNode.get('uid'));
+ try {
+ const data = await fs.promises.readFile(path);
+ return data;
+ } catch ( e ) {
+ this.errors.report('file_cache:read_error', {
+ source: e,
+ trace: true,
+ alarm: true,
+ });
+ }
+ }
+
+ this.errors.report('file_cache:unexpected-cache-state', {
+ message: `Unexpected cache state: ${tracker.phase?.label}`,
+ trace: true,
+ alarm: true,
+ extra: {
+ phase: tracker.phase?.label,
+ }
+ });
+
+ return null;
+ }
+
+ async maybe_store (fsNode, stream) {
+ const size = await fsNode.get('size');
+
+ // If the file is too big, don't cache it
+ if (size > this.disk_max_size) {
+ return { cached: false };
+ }
+
+ const key = await fsNode.get('uid');
+
+ // If the file is already cached, don't cache it again
+ if (this.uid_to_tracker.has(key)) {
+ return { cached: true };
+ }
+
+ // Add file tracker
+ const tracker = new FileTracker({ key, size });
+ this.uid_to_tracker.set(key, tracker);
+ tracker.touch();
+
+
+ // Store binary data in memory (precache)
+ const data = Buffer.alloc(size);
+
+ const [replace_stream, store_stream] = pausing_tee(stream, 2);
+
+ (async () => {
+ let offset = 0;
+ for await (const chunk of store_stream) {
+ chunk.copy(data, offset);
+ offset += chunk.length;
+ }
+
+ await this._precache_make_room(size);
+ console.log(`precache input key: ${key}`);
+ this.precache.set(key, data);
+ tracker.phase = FileTracker.PHASE_PRECACHE;
+ })()
+
+ return { cached: true, stream: replace_stream };
+ }
+
+ async invalidate (fsNode) {
+ const key = await fsNode.get('uid');
+ if ( ! this.uid_to_tracker.has(key) ) return;
+ const tracker = this.uid_to_tracker.get(key);
+ if ( tracker.phase === FileTracker.PHASE_PRECACHE ) {
+ this.precache.delete(key);
+ }
+ if ( tracker.phase === FileTracker.PHASE_DISK ) {
+ await this._disk_evict(tracker);
+ }
+ this.uid_to_tracker.delete(key);
+ }
+
+ async _precache_make_room (size) {
+ if (this._precache_used + size > this.precache_size) {
+ await this._precache_evict(
+ this._precache_used + size - this.precache_size
+ );
+ }
+ }
+
+ async _precache_evict (capacity_needed) {
+ // Sort by score from tracker
+ const sorted = Array.from(this.uid_to_tracker.values())
+ .sort((a, b) => b.score - a.score);
+
+ let capacity = 0;
+ for (const tracker of sorted) {
+ if (tracker.phase !== FileTracker.PHASE_PRECACHE) continue;
+ capacity += tracker.size;
+ await this._maybe_promote_to_disk(tracker);
+ if (capacity >= capacity_needed) break;
+ }
+ }
+
+ async _maybe_promote_to_disk (tracker) {
+ if (tracker.phase !== FileTracker.PHASE_PRECACHE) return;
+
+ // It's important to check that the score of this file is
+ // higher than the combined score of the N files that
+ // would be evicted to make room for it.
+ const sorted = Array.from(this.uid_to_tracker.values())
+ .sort((a, b) => b.score - a.score);
+
+ let capacity = 0;
+ let score_needed = 0;
+ const capacity_needed = this._disk_used + tracker.size - this.disk_limit;
+ for (const tracker of sorted) {
+ if (tracker.phase !== FileTracker.PHASE_DISK) continue;
+ capacity += tracker.size;
+ score_needed += tracker.score;
+ if (capacity >= capacity_needed) break;
+ }
+
+ if (tracker.score < score_needed) return;
+
+ // Now we can remove the lowest scoring files
+ // to make room for this file.
+ capacity = 0;
+ for (const tracker of sorted) {
+ if (tracker.phase !== FileTracker.PHASE_DISK) continue;
+ capacity += tracker.size;
+ await this._disk_evict(tracker);
+ if (capacity >= capacity_needed) break;
+ }
+
+ const { fs } = this.modules;
+ const path = this._get_path(tracker.key);
+ console.log(`precache fetch key I guess?`, tracker.key);
+ const data = this.precache.get(tracker.key);
+ console.log(`path and data: ${path} ${data}`);
+ await fs.promises.writeFile(path, data);
+ this.precache.delete(tracker.key);
+ tracker.phase = FileTracker.PHASE_DISK;
+ }
+
+ async _disk_evict (tracker) {
+ if (tracker.phase !== FileTracker.PHASE_DISK) return;
+
+ const { fs } = this.modules;
+ const path = this._get_path(tracker.key);
+
+ await fs.promises.unlink(path);
+ tracker.phase = FileTracker.PHASE_GONE;
+ this.uid_to_tracker.delete(tracker.key);
+ }
+
+ _register_commands (commands) {
+ commands.registerCommands('fsc', [
+ {
+ id: 'status',
+ handler: async (args, log) => {
+ const { fs } = this.modules;
+ const path = this._get_path('status');
+
+ const status = {
+ precache: {
+ used: this._precache_used,
+ max: this.precache_size,
+ },
+ disk: {
+ used: this._disk_used,
+ max: this.disk_limit,
+ },
+ };
+
+ log.log(JSON.stringify(status, null, 2));
+ }
+ }
+ ]);
+ }
+}
+
+module.exports = {
+ FileCacheService
+};
diff --git a/packages/backend/src/services/file-cache/FileTracker.js b/packages/backend/src/services/file-cache/FileTracker.js
new file mode 100644
index 00000000..d3a593a4
--- /dev/null
+++ b/packages/backend/src/services/file-cache/FileTracker.js
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+/**
+ * FileTracker
+ *
+ * Tracks information about cached files for LRU and LFU eviction.
+ */
+class FileTracker {
+ static PHASE_PENDING = { label: 'pending' };
+ static PHASE_PRECACHE = { label: 'precache' };
+ static PHASE_DISK = { label: 'disk' };
+ static PHASE_GONE = { label: 'gone' };
+
+ constructor ({ key, size }) {
+ this.phase = this.constructor.PHASE_PENDING;
+ this.access_count = 0;
+ this.last_access = 0;
+ this.size = size;
+ this.key = key;
+ this.birth = Date.now();
+ }
+
+ get score () {
+ const weight_recency = 0.5;
+ const weight_access_count = 0.5;
+
+ const recency = Date.now() - this.last_access;
+ const access_count = this.access_count;
+
+ return (weight_access_count * access_count) /
+ (weight_recency * recency);
+ }
+
+ get age () {
+ return Date.now() - this.birth;
+ }
+
+
+ touch () {
+ this.access_count++;
+ this.last_access = Date.now();
+ }
+}
+
+module.exports = {
+ FileTracker
+}
diff --git a/packages/backend/src/services/fs/FSLockService.js b/packages/backend/src/services/fs/FSLockService.js
new file mode 100644
index 00000000..83bf4988
--- /dev/null
+++ b/packages/backend/src/services/fs/FSLockService.js
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { RWLock } = require("../../util/lockutil");
+const { TeePromise } = require("../../util/promise");
+const BaseService = require("../BaseService");
+
+const MODE_READ = Symbol('read');
+const MODE_WRITE = Symbol('write');
+
+class FSLockService extends BaseService {
+ async _construct () {
+ this.locks = {};
+ }
+ async _init () {
+ const svc_commands = this.services.get('commands');
+ svc_commands.registerCommands('fslock', [
+ {
+ id: 'locks',
+ description: 'lists locks',
+ handler: async (args, log) => {
+ for ( const path in this.locks ) {
+ let line = path + ': ';
+ if ( this.locks[path].effective_mode === MODE_READ ) {
+ line += `READING (${this.locks[path].readers_})`;
+ log.log(line);
+ }
+ else
+ if ( this.locks[path].effective_mode === MODE_WRITE ) {
+ line += 'WRITING';
+ log.log(line);
+ }
+ else {
+ line += 'UNKNOWN';
+ log.log(line);
+
+ // log the lock's internal state
+ const lines = JSON.stringify(
+ this.locks[path],
+ null, 2
+ ).split('\n');
+ for ( const line of lines ) {
+ log.log(' -> ' + line);
+ }
+ }
+ }
+ }
+ }
+ ]);
+ }
+ async lock_child (path, name, mode) {
+ if ( path.endsWith('/') ) path = path.slice(0, -1);
+ return await this.lock_path(path + '/' + name, mode);
+ }
+ async lock_path (path, mode) {
+ // TODO: Why???
+ // if ( this.locks === undefined ) this.locks = {};
+
+ if ( ! this.locks[path] ) {
+ const rwlock = new RWLock();
+ rwlock.on_empty_ = () => {
+ delete this.locks[path];
+ };
+ this.locks[path] = rwlock;
+ }
+
+ this.log.noticeme('WAITING FOR LOCK: ' + path + ' ' +
+ mode.toString());
+
+ if ( mode === MODE_READ ) {
+ return await this.locks[path].rlock();
+ }
+
+ if ( mode === MODE_WRITE ) {
+ return await this.locks[path].wlock();
+ }
+
+ throw new Error('Invalid mode');
+ }
+}
+
+module.exports = {
+ MODE_READ,
+ MODE_WRITE,
+ FSLockService
+};
diff --git a/packages/backend/src/services/information/InformationService.js b/packages/backend/src/services/information/InformationService.js
new file mode 100644
index 00000000..db4eda91
--- /dev/null
+++ b/packages/backend/src/services/information/InformationService.js
@@ -0,0 +1,178 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class InformationProvider {
+ constructor (informationService, input) {
+ this.informationService = informationService;
+ this.input = input;
+ }
+
+ provide (output) {
+ this.output = output;
+ return this;
+ }
+
+ addStrategy (id, provider) {
+ this.informationService.register_provider_(
+ this.output, this.input, { id, fn: provider });
+ return this;
+ }
+}
+
+
+class InformationObtainer {
+ constructor (informationService, input) {
+ this.informationService = informationService;
+ this.input = input;
+ }
+
+ obtain (output) {
+ this.output = output;
+ return this;
+ }
+
+ async exec (...args) {
+ const services = this.informationService.services;
+ const traces = services.get('traceService');
+ return await traces.spanify(`OBTAIN ${this.output} FROM ${this.input}`, async () => {
+ return (await this.informationService.obtain_(
+ this.output, this.input, ...args)).result;
+ });
+ }
+}
+
+/**
+ * Allows services to provide methods for obtaining information,
+ * and other services to obtain that information. Also optimizes
+ * obtaining information by determining which methods are the
+ * most efficient for obtaining the information.
+ *
+ * @example Obtain an fsentry given a path:
+ *
+ * const infosvc = services.get('information');
+ * const fsentry = await infosvc
+ * .with('fs.fsentry:path').obtain('fs.fsentry')
+ * .exec(path);
+ *
+ * @example Register a method for obtaining an fsentry given a path:
+ *
+ * const infosvc = services.get('information');
+ * infosvc.given('fs.fsentry:path').provide('fs.fsentry')
+ * .addStrategy(async path => {
+ * // code to obtain fsentry from path
+ * });
+ */
+class InformationService {
+ constructor ({ services }) {
+ this.providers_ = {};
+ this.services = services;
+
+ this.log = services.get('log-service').create('information-service');
+
+ (async () => {
+ await services.ready;
+ if ( services.has('commands') ) {
+ this._register_commands(services.get('commands'));
+ }
+ })();
+ }
+
+ given (input) {
+ return new InformationProvider(this, input);
+ }
+
+ with (input) {
+ return new InformationObtainer(this, input);
+ }
+
+ register_provider_ (output, input, provider) {
+ this.providers_ = this.providers_ || {};
+ this.providers_[output] = this.providers_[output] || {};
+ this.providers_[output][input] = this.providers_[output][input] || [];
+ this.providers_[output][input].push(provider);
+ }
+
+ async obtain_ (output, input, ...args) {
+ const providers = this.providers_[output][input];
+ if ( ! providers ) {
+ throw new Error(`no providers for ${output} <- ${input}`);
+ }
+
+ // shuffle providers (for future performance optimization)
+ providers.sort(() => Math.random() - 0.5);
+
+ // put providers with id 'redis' first
+ providers.sort((a, b) => {
+ if ( a.id === 'redis' ) return -1;
+ if ( b.id === 'redis' ) return 1;
+ return 0;
+ });
+
+ // for now, go with the first provider that provides something
+ for ( const provider of providers ) {
+ this.log.debug(`trying provider ${provider.id} for ${output} <- ${input}`);
+ const result = await provider.fn(...args);
+ this.log.debug(`provider ${provider.id} for ${output} <- ${input} returned ${result}`);
+ // TODO: log strategy used as span attribute/tag
+ if ( result !== undefined ) return { provider: provider.id, result };
+ }
+ }
+
+ _register_commands (commands) {
+ commands.registerCommands('info', [
+ {
+ id: 'providers',
+ description: 'List information providers',
+ handler: async (args, log) => {
+ const providers = this.providers_;
+ for ( const [output, inputs] of Object.entries(providers) ) {
+ for ( const [input, providers] of Object.entries(inputs) ) {
+ for ( const provider of providers ) {
+ log.log(`${output} <- ${input} (${provider.id})`);
+ }
+ }
+ }
+ }
+ },
+ {
+ id: 'get',
+ description: 'List information providers',
+ handler: async (args, log) => {
+ if ( args.length < 1 ) {
+ log.log(`usage: info:get `);
+ return;
+ }
+ const [want, have, value] = args;
+ this.log.debug(`info:get ${want} <- ${have} (${value})`);
+ const result = await this.obtain_(want, have, value);
+ let result_str;
+ try {
+ result_str = JSON.stringify(result.result);
+ } catch (e) {
+ result_str = '' + result.result;
+ }
+ log.log(`${want} <- ${have} (${value}) = ${result_str} (via ${result.provider})`);
+ }
+ }
+ ]);
+ }
+}
+
+module.exports = {
+ InformationService
+};
\ No newline at end of file
diff --git a/packages/backend/src/services/periodic/FSEntryMigrateService.js b/packages/backend/src/services/periodic/FSEntryMigrateService.js
new file mode 100644
index 00000000..ad21b5d4
--- /dev/null
+++ b/packages/backend/src/services/periodic/FSEntryMigrateService.js
@@ -0,0 +1,297 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const seedrandom = require("seedrandom");
+const { id2path, get_user } = require("../../helpers");
+const { generate_random_code } = require("../../util/identifier");
+const { DB_MODE_WRITE } = require("../MysqlAccessService");
+const { DB_MODE_READ } = require("../MysqlAccessService");
+
+class Job {
+ static STATE_GREEN = {};
+ static STATE_YELLOW = {};
+ static STATE_RED = {};
+ constructor ({ dbrr, dbrw, log }) {
+ this.dbrr = dbrr;
+ this.dbrw = dbrw;
+ this.log = log;
+ this.state = this.constructor.STATE_RED;
+ }
+ maybe_stop_ () {
+ if ( this.state !== this.constructor.STATE_GREEN ) {
+ this.log.info(`Stopping job`);
+ this.state = this.constructor.STATE_RED;
+ return true;
+ }
+ return false;
+ }
+ stop () {
+ this.state = this.constructor.STATE_YELLOW;
+ }
+ set_progress (progress) {
+ let bar = '';
+ const WIDTH = 30;
+ const N = Math.floor(WIDTH * progress);
+ for ( let i = 0 ; i < WIDTH ; i++ ) {
+ if ( i < N ) {
+ bar += '=';
+ } else {
+ bar += ' ';
+ }
+ }
+ this.log.info(`${this.constructor.name} :: [${bar}] ${progress.toFixed(2)}%`);
+ }
+}
+
+class Mig_StorePath extends Job {
+ async start (args) {
+ this.state = this.constructor.STATE_GREEN;
+ const { dbrr, dbrw, log } = this;
+
+ for ( ;; ) {
+ const t_0 = performance.now();
+ const [fsentries] = await dbrr.promise().execute(
+ `SELECT id, uuid FROM fsentries WHERE path IS NULL ORDER BY accessed DESC LIMIT 50`
+ );
+
+ if ( fsentries.length === 0 ) {
+ log.info(`No more fsentries to migrate`);
+ this.state = this.constructor.STATE_RED;
+ return;
+ }
+ log.info(`Running migration on ${fsentries.length} fsentries`);
+
+ for ( let i=0 ; i < fsentries.length ; i++ ) {
+ const fsentry = fsentries[i];
+ let path;
+ try {
+ path = await id2path(fsentry.uuid);
+ } catch (e) {
+ // This happens when an fsentry has a missing parent
+ log.error(e);
+ continue;
+ }
+ if ( args.includes('--verbose') ) {
+ log.info(`id=${fsentry.id} uuid=${fsentry.uuid} path=${path}`);
+ }
+ await dbrw.promise().execute(
+ `UPDATE fsentries SET path=? WHERE id=?`,
+ [path, fsentry.id],
+ );
+ }
+
+ const t_1 = performance.now();
+
+ // Give the server a break for twice the time it took to execute the query,
+ // or 100ms at least.
+ const time_to_wait = Math.max(100, 2 * (t_1 - t_0));
+
+ if ( this.maybe_stop_() ) return;
+
+ log.info(`Waiting for ${time_to_wait.toFixed(2)}ms`);
+ await new Promise(rslv => setTimeout(rslv, time_to_wait));
+
+ if ( this.maybe_stop_() ) return;
+ }
+ }
+}
+
+class Mig_IndexAccessed extends Job {
+ async start (args) {
+ this.state = this.constructor.STATE_GREEN;
+ const { dbrr, dbrw, log } = this;
+
+ for ( ;; ) {
+ log.info(`Running update statement`);
+ const t_0 = performance.now();
+ const [results] = await dbrr.promise().execute(
+ `UPDATE fsentries SET accessed = COALESCE(accessed, created) WHERE accessed IS NULL LIMIT 10000`
+ );
+ log.info(`Updated ${results.affectedRows} rows`);
+
+ if ( results.affectedRows === 0 ) {
+ log.info(`No more fsentries to migrate`);
+ this.state = this.constructor.STATE_RED;
+ return;
+ }
+
+ const t_1 = performance.now();
+
+ // Give the server a break for twice the time it took to execute the query,
+ // or 100ms at least.
+ const time_to_wait = Math.max(100, 2 * (t_1 - t_0));
+
+ if ( this.maybe_stop_() ) return;
+
+ log.info(`Waiting for ${time_to_wait.toFixed(2)}ms`);
+ await new Promise(rslv => setTimeout(rslv, time_to_wait));
+
+ if ( this.maybe_stop_() ) return;
+ }
+ }
+}
+
+class Mig_FixTrash extends Job {
+ async start (args) {
+ const { v4: uuidv4 } = require('uuid');
+
+ this.state = this.constructor.STATE_GREEN;
+ const { dbrr, dbrw, log } = this;
+
+ const SQL_NOTRASH_USERS = `
+ SELECT parent.name, parent.uuid FROM fsentries AS parent
+ WHERE parent_uid IS NULL
+ AND NOT EXISTS (
+ SELECT 1 FROM fsentries AS child
+ WHERE child.parent_uid = parent.uuid
+ AND child.name = 'Trash'
+ )
+ `;
+
+ let [user_dirs] = await dbrr.promise().execute(SQL_NOTRASH_USERS);
+
+ for ( const { name, uuid } of user_dirs ) {
+ const username = name;
+ const user_dir_uuid = uuid;
+
+ const t_0 = performance.now();
+ const user = await get_user({ username });
+ const trash_uuid = uuidv4();
+ const trash_ts = Date.now()/1000;
+ log.info(`Fixing trash for user ${user.username} ${user.id} ${user_dir_uuid} ${trash_uuid} ${trash_ts}`);
+
+ const insert_res = await dbrw.promise().execute(`
+ INSERT INTO fsentries
+ (uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable)
+ VALUES
+ ( ?, ?, ?, ?, ?, true, ?, ?, true)
+ `, [trash_uuid, user_dir_uuid, user.id, 'Trash', '/Trash', trash_ts, trash_ts]);
+ log.info(`Inserted ${insert_res[0].affectedRows} rows in fsentries`);
+ // Update uuid cached in the user table
+ const update_res = await dbrw.promise().execute(`
+ UPDATE user SET trash_uuid=? WHERE username=?
+ `, [trash_uuid, user.username]);
+ log.info(`Updated ${update_res[0].affectedRows} rows in user`);
+ const t_1 = performance.now();
+
+ const time_to_wait = Math.max(100, 2 * (t_1 - t_0));
+
+ if ( this.maybe_stop_() ) return;
+
+ log.info(`Waiting for ${time_to_wait.toFixed(2)}ms`);
+ await new Promise(rslv => setTimeout(rslv, time_to_wait));
+
+ if ( this.maybe_stop_() ) return;
+ }
+ }
+}
+
+class Mig_AddReferralCodes extends Job {
+ async start (args) {
+ this.state = this.constructor.STATE_GREEN;
+ const { dbrr, dbrw, log } = this;
+
+ let existing_codes = new Set();
+ const SQL_EXISTING_CODES = `SELECT referral_code FROM user`;
+ let [codes] = await dbrr.promise().execute(SQL_EXISTING_CODES);
+ for ( const { referal_code } of codes ) {
+ existing_codes.add(referal_code);
+ }
+
+ const SQL_USER_IDS = `SELECT id, referral_code FROM user`;
+
+ let [users] = await dbrr.promise().execute(SQL_USER_IDS);
+
+ let i = 0;
+
+ for ( const user of users ) {
+ if ( user.referal_code ) continue;
+ // create seed for deterministic random value
+ let iteration = 0;
+ let rng = seedrandom(`gen1-${user.id}`);
+ let referal_code = generate_random_code(8, { rng });
+
+ while ( existing_codes.has(referal_code) ) {
+ rng = seedrandom(`gen1-${user.id}-${++iteration}`);
+ referal_code = generate_random_code(8, { rng });
+ }
+
+ const update_res = await dbrw.promise().execute(`
+ UPDATE user SET referral_code=? WHERE id=?
+ `, [referal_code, user.id]);
+
+ i++;
+ if ( i % 500 == 0 ) this.set_progress(i / users.length);
+
+ if ( this.maybe_stop_() ) return;
+ }
+ }
+}
+
+class Mig_AuditInitialStorage extends Job {
+ async start (args) {
+ this.state = this.constructor.STATE_GREEN;
+ const { dbrr, dbrw, log } = this;
+
+ // TODO: this migration will add an audit log for each user's
+ // storage capacity before auditing was implemented.
+ }
+}
+
+class FSEntryMigrateService {
+ constructor ({ services }) {
+ const mysql = services.get('mysql');
+ const dbrr = mysql.get(DB_MODE_READ, 'fsentry-migrate');
+ const dbrw = mysql.get(DB_MODE_WRITE, 'fsentry-migrate');
+ const log = services.get('log-service').create('fsentry-migrate');
+
+ const migrations = {
+ 'store-path': new Mig_StorePath({ dbrr, dbrw, log }),
+ 'index-accessed': new Mig_IndexAccessed({ dbrr, dbrw, log }),
+ 'fix-trash': new Mig_FixTrash({ dbrr, dbrw, log }),
+ 'gen-referral-codes': new Mig_AddReferralCodes({ dbrr, dbrw, log }),
+ };
+
+ services.get('commands').registerCommands('fsentry-migrate', [
+ {
+ id: 'start',
+ description: 'start a migration',
+ handler: async (args, log) => {
+ const [migration] = args;
+ if ( ! migrations[migration] ) {
+ throw new Error(`unknown migration: ${migration}`);
+ }
+ migrations[migration].start(args.slice(1));
+ }
+ },
+ {
+ id: 'stop',
+ description: 'stop a migration',
+ handler: async (args, log) => {
+ const [migration] = args;
+ if ( ! migrations[migration] ) {
+ throw new Error(`unknown migration: ${migration}`);
+ }
+ migrations[migration].stop();
+ }
+ }
+ ]);
+ }
+}
+
+module.exports = { FSEntryMigrateService };
diff --git a/packages/backend/src/services/runtime-analysis/AlarmService.js b/packages/backend/src/services/runtime-analysis/AlarmService.js
new file mode 100644
index 00000000..d3c3eb78
--- /dev/null
+++ b/packages/backend/src/services/runtime-analysis/AlarmService.js
@@ -0,0 +1,408 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const JSON5 = require('json5');
+const seedrandom = require('seedrandom');
+
+const util = require('util');
+const _path = require('path');
+const fs = require('fs');
+
+const { fallbackRead } = require('../../util/files.js');
+const { generate_identifier } = require('../../util/identifier.js');
+const { stringify_log_entry } = require('./LogService.js');
+const BaseService = require('../BaseService.js');
+const { split_lines } = require('../../util/stdioutil.js');
+
+class AlarmService extends BaseService {
+ async _construct () {
+ this.alarms = {};
+ this.alarm_aliases = {};
+
+ this.known_errors = [];
+ }
+ async _init () {
+ const services = this.services;
+ this.pager = services.get('pager');
+
+ // TODO:[self-hosted] fix this properly
+ this.known_errors = [];
+ // (async () => {
+ // try {
+ // this.known_errors = JSON5.parse(
+ // await fallbackRead(
+ // 'data/known_errors.json5',
+ // '/var/puter/data/known_errors.json5',
+ // ),
+ // );
+ // } catch (e) {
+ // this.create(
+ // 'missing-known-errors',
+ // e.message,
+ // )
+ // }
+ // })();
+
+ this._register_commands(services.get('commands'));
+
+ if ( this.global_config.env === 'dev' ) {
+ this.alarm_widget = () => {
+ // return `\x1B[31;1m alarms (${
+ // Object.keys(this.alarms)
+ // })\x1B[0m`;
+ const lines = [];
+ for ( const alarm of Object.values(this.alarms) ) {
+ const line =
+ `\x1B[31;1m [alarm]\x1B[0m ` +
+ `${alarm.id_string}: ${alarm.message} (${alarm.count})`;
+ const line_lines = split_lines(line);
+ lines.push(...line_lines);
+ }
+
+ return lines;
+ }
+ }
+ }
+
+ adapt_id_ (id) {
+ // let shorten = false;
+ // // Check if id uses characters that aren't on a US QWERTY keyboard.
+ // if ( /[^\x20-\x7E]/.test(id) ) shorten = true;
+
+ // // Check if id is too long
+ // if ( id.length > 20 ) shorten = true;
+ let shorten = true;
+
+ if ( shorten ) {
+ const rng = seedrandom(id);
+ id = generate_identifier('-', rng);
+ }
+
+ return id;
+ }
+
+ create (id, message, fields) {
+ this.log.error(`upcoming alarm: ${id}: ${message}`);
+ let existing = false;
+ const alarm = (() => {
+ const short_id = this.adapt_id_(id);
+
+ if ( this.alarms[id] ) {
+ existing = true;
+ return this.alarms[id];
+ }
+
+ const alarm = this.alarms[id] = this.alarm_aliases[short_id] = {
+ id,
+ short_id,
+ started: Date.now(),
+ occurrences: [],
+ };
+
+ Object.defineProperty(alarm, 'count', {
+ get () {
+ return alarm.timestamps?.length ?? 0;
+ }
+ });
+
+ Object.defineProperty(alarm, 'id_string', {
+ get () {
+ if ( alarm.id.length < 20 ) {
+ return alarm.id;
+ }
+
+ const truncatedLongId = alarm.id.slice(0, 20) + '...';
+
+ return `${alarm.short_id} (${truncatedLongId})`;
+ }
+ });
+
+ return alarm;
+ })();
+
+ const occurance = {
+ message,
+ fields,
+ timestamp: Date.now(),
+ };
+
+ // Keep logs from the previous occurrence if:
+ // - it's one of the first 3 occurrences
+ // - the 10th, 100th, 1000th...etc occurrence
+ if ( alarm.count > 3 && Math.log10(alarm.count) % 1 !== 0 ) {
+ delete alarm.occurrences[alarm.occurrences.length - 1].logs;
+ }
+ occurance.logs = this.log.get_log_buffer();
+
+ alarm.message = message;
+ alarm.fields = { ...alarm.fields, ...fields };
+ alarm.timestamps = (alarm.timestamps ?? []).concat(Date.now());
+ alarm.occurrences.push(occurance);
+
+ if ( fields?.error ) {
+ alarm.error = fields.error;
+ }
+
+ if ( alarm.source ) {
+ console.error(alarm.error);
+ }
+
+ if ( existing ) {
+ this.handle_alarm_repeat_(alarm);
+ } else {
+ this.handle_alarm_on_(alarm);
+ }
+ }
+
+ clear (id) {
+ const alarm = this.alarms[id];
+ if ( !alarm ) {
+ return;
+ }
+ delete this.alarms[id];
+ this.handle_alarm_off_(alarm);
+ }
+
+ apply_known_errors_ (alarm) {
+ const rule_matches = rule => {
+ const match = rule.match;
+ if ( match.id !== alarm.id ) return false;
+ if ( match.message && match.message !== alarm.message ) return false;
+ if ( match.fields ) {
+ for ( const [key, value] of Object.entries(match.fields) ) {
+ if ( alarm.fields[key] !== value ) return false;
+ }
+ }
+ return true;
+ }
+
+ const rule_actions = {
+ 'no-alert': () => alarm.no_alert = true,
+ 'severity': action => alarm.severity = action.value,
+ };
+
+ const apply_action = action => {
+ rule_actions[action.type](action);
+ };
+
+ for ( const rule of this.known_errors ) {
+ if ( rule_matches(rule) ) apply_action(rule.action);
+ }
+ }
+
+
+ handle_alarm_repeat_ (alarm) {
+ this.log.warn(
+ `REPEAT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,
+ alarm.fields,
+ );
+
+ this.apply_known_errors_(alarm);
+
+ if ( alarm.no_alert ) return;
+
+ const severity = alarm.severity ?? 'critical';
+
+ const fields_clean = {};
+ for ( const [key, value] of Object.entries(alarm.fields) ) {
+ fields_clean[key] = util.inspect(value);
+ }
+
+ this.pager.alert({
+ id: (alarm.id ?? 'something-bad') + '-r_${alarm.count}',
+ message: alarm.message ?? alarm.id ?? 'something bad happened',
+ source: 'alarm-service',
+ severity,
+ custom: {
+ fields: fields_clean,
+ trace: alarm.error?.stack,
+ }
+ });
+ }
+
+ handle_alarm_on_ (alarm) {
+ this.log.error(
+ `ACTIVE ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,
+ alarm.fields,
+ );
+
+ this.apply_known_errors_(alarm);
+
+ // dev console
+ if ( this.global_config.env === 'dev' && ! this.attached_dev ) {
+ this.attached_dev = true;
+ const svc_devConsole = this.services.get('dev-console');
+ svc_devConsole.turn_on_the_warning_lights();
+ svc_devConsole.add_widget(this.alarm_widget);
+ }
+
+ if ( alarm.no_alert ) return;
+
+ const severity = alarm.severity ?? 'critical';
+
+ const fields_clean = {};
+ for ( const [key, value] of Object.entries(alarm.fields) ) {
+ fields_clean[key] = util.inspect(value);
+ }
+
+ this.pager.alert({
+ id: alarm.id ?? 'something-bad',
+ message: alarm.message ?? alarm.id ?? 'something bad happened',
+ source: 'alarm-service',
+ severity,
+ custom: {
+ fields: fields_clean,
+ trace: alarm.error?.stack,
+ }
+ });
+
+ // Write a .log file for the alert that happened
+ try {
+ const lines = [];
+ lines.push(`ALERT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`),
+ lines.push(`started: ${new Date(alarm.started).toISOString()}`);
+ lines.push(`short id: ${alarm.short_id}`);
+ lines.push(`original id: ${alarm.id}`);
+ lines.push(`severity: ${severity}`);
+ lines.push(`message: ${alarm.message}`);
+ lines.push(`fields: ${JSON.stringify(fields_clean)}`);
+
+ const alert_info = lines.join('\n');
+
+ (async () => {
+ try {
+ await fs.appendFileSync(`alert_${alarm.id}.log`, alert_info + '\n');
+ } catch (e) {
+ this.log.error(`failed to write alert log: ${e.message}`);
+ }
+ })();
+ } catch (e) {
+ this.log.error(`failed to write alert log: ${e.message}`);
+ }
+ }
+
+ handle_alarm_off_ (alarm) {
+ this.log.info(
+ `CLEAR ${alarm.id} :: ${alarm.message} (${alarm.count})`,
+ alarm.fields,
+ );
+ }
+
+ get_alarm (id) {
+ return this.alarms[id] ?? this.alarm_aliases[id];
+ }
+
+ _register_commands (commands) {
+ commands.registerCommands('alarm', [
+ {
+ id: 'list',
+ description: 'list alarms',
+ handler: async (args, log) => {
+ for ( const alarm of Object.values(this.alarms) ) {
+ log.log(`${alarm.id_string}: ${alarm.message} (${alarm.count})`);
+ }
+ }
+ },
+ {
+ id: 'info',
+ description: 'show info about an alarm',
+ handler: async (args, log) => {
+ const [id] = args;
+ const alarm = this.get_alarm(id);
+ if ( !alarm ) {
+ log.log(`no alarm with id ${id}`);
+ return;
+ }
+ log.log(`\x1B[33;1m${alarm.id_string}\x1B[0m :: ${alarm.message} (${alarm.count})`);
+ log.log(`started: ${new Date(alarm.started).toISOString()}`);
+ log.log(`short id: ${alarm.short_id}`);
+ log.log(`original id: ${alarm.id}`);
+
+ // print stack trace of alarm error
+ if ( alarm.error ) {
+ log.log(alarm.error.stack);
+ }
+ // print other fields
+ for ( const [key, value] of Object.entries(alarm.fields) ) {
+ log.log(`- ${key}: ${util.inspect(value)}`);
+ }
+ }
+ },
+ {
+ id: 'clear',
+ description: 'clear an alarm',
+ handler: async (args, log) => {
+ const [id] = args;
+ const alarm = this.get_alarm(id);
+ if ( ! alarm ) {
+ log.log(
+ `no alarm with id ${id}; ` +
+ `but calling clear(${JSON.stringify(id)}) anyway.`
+ );
+ }
+ this.clear(id);
+ }
+ },
+ {
+ id: 'clear-all',
+ description: 'clear all alarms',
+ handler: async (args, log) => {
+ const alarms = Object.values(this.alarms);
+ this.alarms = {};
+ for ( const alarm of alarms ) {
+ this.handle_alarm_off_(alarm);
+ }
+ }
+ },
+ {
+ id: 'sound',
+ description: 'sound an alarm',
+ handler: async (args, log) => {
+ const [id, message] = args;
+ this.create(id ?? 'test', message, {});
+ }
+ },
+ {
+ id: 'inspect',
+ description: 'show logs that happened an alarm',
+ handler: async (args, log) => {
+ const [id, occurance_idx] = args;
+ const alarm = this.get_alarm(id);
+ if ( !alarm ) {
+ log.log(`no alarm with id ${id}`);
+ return;
+ }
+ const occurance = alarm.occurrences[occurance_idx];
+ if ( !occurance ) {
+ log.log(`no occurance with index ${occurance_idx}`);
+ return;
+ }
+ log.log(`┏━━ Logs before: ${alarm.id_string} ━━━━`);
+ for ( const lg of occurance.logs ) {
+ log.log("┃ " + stringify_log_entry(lg));
+ }
+ log.log(`┗━━ Logs before: ${alarm.id_string} ━━━━`);
+ }
+ },
+ ]);
+ }
+}
+
+module.exports = {
+ AlarmService,
+};
diff --git a/packages/backend/src/services/runtime-analysis/ErrorService.js b/packages/backend/src/services/runtime-analysis/ErrorService.js
new file mode 100644
index 00000000..e864755c
--- /dev/null
+++ b/packages/backend/src/services/runtime-analysis/ErrorService.js
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("../BaseService");
+
+class ErrorContext {
+ constructor (error_service, log_context) {
+ this.error_service = error_service;
+ this.log_context = log_context;
+ }
+ report (location, fields) {
+ fields = {
+ ...fields,
+ logger: this.log_context,
+ };
+ this.error_service.report(location, fields);
+ }
+}
+
+class ErrorService extends BaseService {
+ async init () {
+ const services = this.services;
+ this.alarm = services.get('alarm');
+ this.backupLogger = services.get('log-service').create('error-service');
+ }
+ create (log_context) {
+ return new ErrorContext(this, log_context);
+ }
+ report (location, { source, logger, trace, extra, message }, alarm = true) {
+ message = message ?? source?.message;
+ logger = logger ?? this.backupLogger;
+ logger.error(`Error @ ${location}: ${message}; ` + source?.stack);
+ if ( trace ) {
+ logger.error(source);
+ }
+
+ if ( alarm ) {
+ const alarm_id = `${location}:${message}`;
+ this.alarm.create(alarm_id, message, {
+ error: source,
+ ...extra,
+ });
+ }
+ }
+}
+
+module.exports = { ErrorService };
diff --git a/packages/backend/src/services/runtime-analysis/ExpectationService.js b/packages/backend/src/services/runtime-analysis/ExpectationService.js
new file mode 100644
index 00000000..df24dc2d
--- /dev/null
+++ b/packages/backend/src/services/runtime-analysis/ExpectationService.js
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { v4: uuidv4 } = require('uuid');
+const { quot } = require('../../util/strutil');
+const BaseService = require('../BaseService');
+
+class WorkUnit {
+ static create () {
+ return new WorkUnit();
+ }
+ constructor () {
+ this.id = uuidv4();
+ this.checkpoint_ = null;
+ }
+ checkpoint (label) {
+ console.log('CHECKPOINT', label);
+ this.checkpoint_ = label;
+ }
+}
+
+class CheckpointExpectation {
+ constructor (workUnit, checkpoint) {
+ this.workUnit = workUnit;
+ this.checkpoint = checkpoint;
+ }
+ check () {
+ // TODO: should be true if checkpoint was ever reached
+ return this.workUnit.checkpoint_ == this.checkpoint;
+ }
+ report (log) {
+ if ( this.check() ) return;
+ log.log(
+ `operation(${this.workUnit.id}): ` +
+ `expected ${quot(this.checkpoint)} ` +
+ `and got ${quot(this.workUnit.checkpoint_)}.`
+ );
+ }
+}
+
+/**
+ * This service helps diagnose errors involving the potentially
+ * complex relationships between asynchronous operations.
+ */
+class ExpectationService extends BaseService {
+ async _construct () {
+ this.expectations_ = [];
+ }
+
+ async _init () {
+ const services = this.services;
+
+ // TODO: service to track all interval functions?
+ setInterval(() => {
+ this.purgeExpectations_();
+ }, 1000);
+
+ const commands = services.get('commands');
+ commands.registerCommands('expectations', [
+ {
+ id: 'pending',
+ description: 'lists pending expectations',
+ handler: async (args, log) => {
+ this.purgeExpectations_();
+ if ( this.expectations_.length < 1 ) {
+ log.log(`there are none`);
+ return;
+ }
+ for ( const expectation of this.expectations_ ) {
+ expectation.report(log);
+ }
+ }
+ }
+ ]);
+ }
+
+ purgeExpectations_ () {
+ return;
+ for ( let i=0 ; i < this.expectations_.length ; i++ ) {
+ if ( this.expectations_[i].check() ) {
+ this.expectations_[i] = null;
+ }
+ }
+ this.expectations_ = this.expectations_.filter(v => v !== null);
+ }
+
+ expect_eventually ({ workUnit, checkpoint }) {
+ this.expectations_.push(new CheckpointExpectation(workUnit, checkpoint));
+ }
+}
+
+
+
+module.exports = {
+ WorkUnit,
+ ExpectationService
+};
\ No newline at end of file
diff --git a/packages/backend/src/services/runtime-analysis/HeapMonService.js b/packages/backend/src/services/runtime-analysis/HeapMonService.js
new file mode 100644
index 00000000..30f7c130
--- /dev/null
+++ b/packages/backend/src/services/runtime-analysis/HeapMonService.js
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const memwatch = require('@airbnb/node-memwatch');
+
+class HeapMonService {
+ constructor ({ services, my_config }) {
+ this.log = services.get('log-service').create('heap-monitor');
+ this.alarm = services.get('alarm');
+
+ let hd, hd_ts;
+
+ if ( my_config.heapdiff ) {
+ hd = new memwatch.HeapDiff();
+ hd_ts = Date.now();
+ }
+
+ let heapdiff_interval = my_config.heapdiff_interval ?? 1;
+ heapdiff_interval *= 1000;
+
+ memwatch.on('stats', (stats) => {
+ this.log.info('stats', stats);
+
+ (() => {
+ if ( ! my_config.heapdiff ) return
+
+ const now = Date.now();
+
+ if ( (now - hd_ts) < heapdiff_interval ) return;
+
+ const diff = hd.end();
+ this.log.info('heapdiff', diff);
+ hd = new memwatch.HeapDiff();
+ hd_ts = now;
+ })();
+ });
+
+ memwatch.on('leak', (info) => {
+ this.log.error('leak', info);
+ this.alarm.create('heap-leak', 'memory leak detected', info);
+ });
+ }
+}
+
+module.exports = { HeapMonService };
\ No newline at end of file
diff --git a/packages/backend/src/services/runtime-analysis/LogService.js b/packages/backend/src/services/runtime-analysis/LogService.js
new file mode 100644
index 00000000..4c8b1498
--- /dev/null
+++ b/packages/backend/src/services/runtime-analysis/LogService.js
@@ -0,0 +1,440 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const logSeverity = (ordinal, label, esc, winst) => ({ ordinal, label, esc, winst });
+const LOG_LEVEL_ERRO = logSeverity(0, 'ERRO', '31;1', 'error');
+const LOG_LEVEL_WARN = logSeverity(1, 'WARN', '33;1', 'warn');
+const LOG_LEVEL_INFO = logSeverity(2, 'INFO', '36;1', 'info');
+const LOG_LEVEL_TICK = logSeverity(10, 'TICK', '34;1', 'info');
+const LOG_LEVEL_DEBU = logSeverity(4, 'DEBU', '37;1', 'debug');
+const LOG_LEVEL_NOTICEME = logSeverity(4, 'NOTICE_ME', '33;1', 'error');
+const LOG_LEVEL_SYSTEM = logSeverity(4, 'SYSTEM', '33;1', 'system');
+
+const winston = require('winston');
+const { Context } = require('../../util/context');
+const BaseService = require('../BaseService');
+require('winston-daily-rotate-file');
+
+const WINSTON_LEVELS = {
+ system: 0,
+ error: 1,
+ warn: 10,
+ info: 20,
+ http: 30,
+ verbose: 40,
+ debug: 50,
+ silly: 60
+};
+
+class LogContext {
+ constructor (logService, { crumbs, fields }) {
+ this.logService = logService;
+ this.crumbs = crumbs;
+ this.fields = fields;
+ }
+
+ sub (name, fields = {}) {
+ return new LogContext(
+ this.logService,
+ {
+ crumbs: name ? [...this.crumbs, name] : [...this.crumbs],
+ fields: {...this.fields, ...fields},
+ }
+ );
+ }
+
+ info (message, fields, objects) { this.log(LOG_LEVEL_INFO, message, fields, objects); }
+ warn (message, fields, objects) { this.log(LOG_LEVEL_WARN, message, fields, objects); }
+ debug (message, fields, objects) { this.log(LOG_LEVEL_DEBU, message, fields, objects); }
+ error (message, fields, objects) { this.log(LOG_LEVEL_ERRO, message, fields, objects); }
+ tick (message, fields, objects) { this.log(LOG_LEVEL_TICK, message, fields, objects); }
+ called (fields = {}) {
+ this.log(LOG_LEVEL_DEBU, 'called', fields);
+ }
+ noticeme (message, fields, objects) {
+ this.log(LOG_LEVEL_NOTICEME, message, fields, objects);
+ }
+ system (message, fields, objects) {
+ this.log(LOG_LEVEL_SYSTEM, message, fields, objects);
+ }
+
+ cache (isCacheHit, identifier, fields = {}) {
+ this.log(
+ LOG_LEVEL_DEBU,
+ isCacheHit ? 'cache_hit' : 'cache_miss',
+ { identifier, ...fields },
+ );
+ }
+
+ log (log_level, message, fields = {}, objects = {}) {
+ fields = { ...this.fields, ...fields };
+ {
+ const x = Context.get(undefined, { allow_fallback: true });
+ if ( x && x.get('trace_request') ) {
+ fields.trace_request = x.get('trace_request');
+ }
+ }
+ this.logService.log_(
+ log_level,
+ this.crumbs,
+ message, fields, objects,
+ );
+ }
+
+ // convenience method to get a trace id that isn't as difficult
+ // for a human to read as a uuid.
+ mkid () {
+ // generate trace id
+ const trace_id = [];
+ for ( let i = 0; i < 2; i++ ) {
+ trace_id.push(Math.random().toString(36).slice(2, 8));
+ }
+ return trace_id.join('-');
+ }
+
+ // add a trace id to this logging context
+ traceOn () {
+ this.fields.trace_id = this.mkid();
+ return this;
+ }
+
+ get_log_buffer () {
+ return this.logService.get_log_buffer();
+ }
+}
+
+let log_epoch = Date.now();
+const stringify_log_entry = ({ log_lvl, crumbs, message, fields, objects }) => {
+ let m = `\x1B[${log_lvl.esc}m[${log_lvl.label}\x1B[0m`;
+ for ( const crumb of crumbs ) {
+ m += `::${crumb}`;
+ }
+ m += `\x1B[${log_lvl.esc}m]\x1B[0m`;
+ for ( const k in fields ) {
+ if ( k === 'timestamp' ) continue;
+ let v; try {
+ v = JSON.stringify(fields[k]);
+ } catch (e) {
+ v = '' + fields[k];
+ }
+ m += ` ${k}=${v}`;
+ }
+ if ( fields.timestamp ) {
+ // display seconds since logger epoch
+ const n = (fields.timestamp - log_epoch) / 1000;
+ m += ` (${n.toFixed(3)}s)`;
+ }
+ m += ` ${message}`;
+ return m;
+};
+
+
+class DevLogger {
+ // TODO: this should eventually delegate to winston logger
+ constructor (log, opt_delegate) {
+ this.log = log;
+
+ if ( opt_delegate ) {
+ this.delegate = opt_delegate;
+ }
+ }
+ onLogMessage (log_lvl, crumbs, message, fields, objects) {
+ if ( this.delegate ) {
+ this.delegate.onLogMessage(
+ log_lvl, crumbs, message, fields, objects,
+ );
+ }
+ this.log(stringify_log_entry({
+ log_lvl, crumbs, message, fields, objects,
+ }));
+ }
+}
+
+class NullLogger {
+ // TODO: this should eventually delegate to winston logger
+ constructor (log, opt_delegate) {
+ this.log = log;
+
+ if ( opt_delegate ) {
+ this.delegate = opt_delegate;
+ }
+ }
+ onLogMessage () {
+ }
+}
+
+class WinstonLogger {
+ constructor (winst) {
+ this.winst = winst;
+ }
+ onLogMessage (log_lvl, crumbs, message, fields, objects) {
+ this.winst.log({
+ ...fields,
+ label: crumbs.join('.'),
+ level: log_lvl.winst,
+ message,
+ });
+ }
+}
+
+class TimestampLogger {
+ constructor (delegate) {
+ this.delegate = delegate;
+ }
+ onLogMessage (log_lvl, crumbs, message, fields, ...a) {
+ fields = { ...fields, timestamp: new Date() };
+ this.delegate.onLogMessage(log_lvl, crumbs, message, fields, ...a);
+ }
+}
+
+class BufferLogger {
+ constructor (size, delegate) {
+ this.size = size;
+ this.delegate = delegate;
+ this.buffer = [];
+ }
+ onLogMessage (log_lvl, crumbs, message, fields, ...a) {
+ this.buffer.push({ log_lvl, crumbs, message, fields, ...a });
+ if ( this.buffer.length > this.size ) {
+ this.buffer.shift();
+ }
+ this.delegate.onLogMessage(log_lvl, crumbs, message, fields, ...a);
+ }
+}
+
+class CustomLogger {
+ constructor (delegate, callback) {
+ this.delegate = delegate;
+ this.callback = callback;
+ }
+ onLogMessage (log_lvl, crumbs, message, fields, ...a) {
+ // Logging is allowed to be performed without a context, but we
+ // don't want log functions to be asynchronous which rules out
+ // wrapping with Context.allow_fallback. Instead we provide a
+ // context as a parameter.
+ const context = Context.get(undefined, { allow_fallback: true });
+
+ const {
+ log_lvl: _log_lvl,
+ crumbs: _crumbs,
+ message: _message,
+ fields: _fields,
+ args,
+ } = this.callback({
+ context,
+ log_lvl, crumbs, message, fields, args: a,
+ });
+ this.delegate.onLogMessage(
+ _log_lvl ?? log_lvl,
+ _crumbs ?? crumbs,
+ _message ?? message,
+ _fields ?? fields,
+ ...(args ?? a ?? []),
+ );
+ }
+}
+
+class LogService extends BaseService {
+ static MODULES = {
+ path: require('path'),
+ }
+ async _construct () {
+ this.loggers = [];
+ this.bufferLogger = null;
+ }
+ register_log_middleware (callback) {
+ this.loggers[0] = new CustomLogger(this.loggers[0], callback);
+ }
+ async _init () {
+ const config = this.global_config;
+
+ this.ensure_log_directory_();
+
+ let logger;
+
+ logger = new WinstonLogger(
+ winston.createLogger({
+ levels: WINSTON_LEVELS,
+ transports: [
+ new winston.transports.DailyRotateFile({
+ filename: `${this.log_directory}/%DATE%.log`,
+ datePattern: 'YYYY-MM-DD',
+ zippedArchive: true,
+ maxSize: '20m',
+
+ // TODO: uncomment when we have a log backup strategy
+ // maxFiles: '14d',
+ }),
+ new winston.transports.DailyRotateFile({
+ level: 'error',
+ filename: `${this.log_directory}/error-%DATE%.log`,
+ datePattern: 'YYYY-MM-DD',
+ zippedArchive: true,
+ maxSize: '20m',
+
+ // TODO: uncomment when we have a log backup strategy
+ // maxFiles: '14d',
+ }),
+ new winston.transports.DailyRotateFile({
+ level: 'system',
+ filename: `${this.log_directory}/system-%DATE%.log`,
+ datePattern: 'YYYY-MM-DD',
+ zippedArchive: true,
+ maxSize: '20m',
+
+ // TODO: uncomment when we have a log backup strategy
+ // maxFiles: '14d',
+ }),
+ ],
+ }),
+ );
+
+ if ( config.env === 'dev' ) {
+ logger = config.flag_no_logs // useful for profiling
+ ? new NullLogger()
+ : new DevLogger(console.log.bind(console), logger);
+ }
+
+ logger = new TimestampLogger(logger);
+
+ logger = new BufferLogger(config.log_buffer_size ?? 20, logger);
+ this.bufferLogger = logger;
+
+ this.loggers.push(logger);
+
+ this.output_lvl = LOG_LEVEL_DEBU;
+ if ( config.logger ) {
+ // config.logger.level is a string, e.g. 'debug'
+
+ // first we find the appropriate log level
+ const output_lvl = Object.values({
+ LOG_LEVEL_ERRO,
+ LOG_LEVEL_WARN,
+ LOG_LEVEL_INFO,
+ LOG_LEVEL_DEBU,
+ LOG_LEVEL_TICK,
+ }).find(lvl => {
+ return lvl.label === config.logger.level.toUpperCase() ||
+ lvl.winst === config.logger.level.toLowerCase() ||
+ lvl.ordinal === config.logger.level;
+ });
+
+ // then we set the output level to the ordinal of that level
+ this.output_lvl = output_lvl.ordinal;
+ }
+
+ this.log = this.create('log-service');
+ this.log.system('log service started', {
+ output_lvl: this.output_lvl,
+ log_directory: this.log_directory,
+ });
+
+ this.services.logger = this.create('services-container');
+ }
+
+ create (prefix, fields = {}) {
+ const logContext = new LogContext(
+ this,
+ {
+ crumbs: [prefix],
+ fields,
+ },
+ );
+
+ return logContext;
+ }
+
+ log_ (log_lvl, crumbs, message, fields, objects) {
+ try {
+ // skip messages that are above the output level
+ if ( log_lvl.ordinal > this.output_lvl ) return;
+
+ for ( const logger of this.loggers ) {
+ logger.onLogMessage(
+ log_lvl, crumbs, message, fields, objects,
+ );
+ }
+ } catch (e) {
+ // If logging fails, we don't want anything to happen
+ // that might trigger a log message. This causes an
+ // infinite loop and I learned that the hard way.
+ console.error('Logging failed', e);
+
+ // TODO: trigger an alarm either in a non-logging
+ // context (prereq: per-context service overrides)
+ // or with a cooldown window (prereq: cooldowns in AlarmService)
+ }
+ }
+
+ ensure_log_directory_ () {
+ // STEP 1: Try /var/puter/logs/heyputer
+ {
+ const fs = require('fs');
+ const path = '/var/puter/logs/heyputer';
+ try {
+ fs.mkdirSync(path, { recursive: true });
+ this.log_directory = path;
+ return;
+ } catch (e) {
+ // ignore
+ }
+ }
+
+ // STEP 2: Try /tmp/heyputer
+ {
+ const fs = require('fs');
+ const path = '/tmp/heyputer';
+ try {
+ fs.mkdirSync(path, { recursive: true });
+ this.log_directory = path;
+ return;
+ } catch (e) {
+ // ignore
+ }
+ }
+
+ // STEP 3: Try working directory
+ {
+ const fs = require('fs');
+ const path = './heyputer';
+ try {
+ fs.mkdirSync(path, { recursive: true });
+ this.log_directory = path;
+ return;
+ } catch (e) {
+ // ignore
+ }
+ }
+
+ // STEP 4: Give up
+ throw new Error('Unable to create or find log directory');
+ }
+
+ get_log_file (name) {
+ return this.modules.path.join(this.log_directory, name);
+ }
+
+ get_log_buffer () {
+ return this.bufferLogger.buffer;
+ }
+}
+
+module.exports = {
+ LogService,
+ stringify_log_entry
+};
\ No newline at end of file
diff --git a/packages/backend/src/services/runtime-analysis/PagerService.js b/packages/backend/src/services/runtime-analysis/PagerService.js
new file mode 100644
index 00000000..ccdc304f
--- /dev/null
+++ b/packages/backend/src/services/runtime-analysis/PagerService.js
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const pdjs = require('@pagerduty/pdjs');
+const BaseService = require('../BaseService');
+
+class PagerService extends BaseService {
+ async _construct () {
+ this.config = this.global_config.pager;
+ this.alertHandlers_ = [];
+
+ }
+ async _init () {
+ const services = this.services;
+
+ this.alertHandlers_ = [];
+
+ if ( ! this.config ) {
+ return;
+ }
+
+ this.onInit();
+
+ this._register_commands(services.get('commands'));
+ }
+
+ onInit () {
+ if ( this.config.pagerduty && this.config.pagerduty.enabled ) {
+ this.alertHandlers_.push(async alert => {
+ const event = pdjs.event;
+
+ const fields_clean = {};
+ for ( const [key, value] of Object.entries(alert?.fields ?? {}) ) {
+ fields_clean[key] = util.inspect(value);
+ }
+
+ this.log.info('it is sending to PD');
+ await event({
+ data: {
+ routing_key: this.config.pagerduty.routing_key,
+ event_action: 'trigger',
+ dedup_key: alert.id,
+ payload: {
+ summary: alert.message,
+ source: alert.source,
+ severity: alert.severity,
+ custom_details: alert.custom,
+ },
+ },
+ });
+ });
+ }
+ }
+
+ async alert (alert) {
+ for ( const handler of this.alertHandlers_ ) {
+ try {
+ await handler(alert);
+ } catch (e) {
+ this.log.error(`failed to send pager alert: ${e?.message}`);
+ }
+ }
+ }
+
+ _register_commands (commands) {
+ commands.registerCommands('pager', [
+ {
+ id: 'test-alert',
+ description: 'create a test alert',
+ handler: async (args, log) => {
+ const [severity] = args;
+ await this.alert({
+ id: 'test-alert',
+ message: 'test alert',
+ source: 'test',
+ severity,
+ });
+ }
+ }
+ ])
+ }
+
+}
+
+module.exports = {
+ PagerService,
+};
diff --git a/packages/backend/src/services/runtime-analysis/ProcessEventService.js b/packages/backend/src/services/runtime-analysis/ProcessEventService.js
new file mode 100644
index 00000000..30e73fc0
--- /dev/null
+++ b/packages/backend/src/services/runtime-analysis/ProcessEventService.js
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Context } = require("../../util/context");
+
+class ProcessEventService {
+ constructor ({ services }) {
+ const log = services.get('log-service').create('process-event-service');
+ const errors = services.get('error-service').create(log);
+
+ // TODO: when the service lifecycle is implemented, but these
+ // in the init hook
+
+ process.on('uncaughtException', async (err, origin) => {
+ await Context.allow_fallback(async () => {
+ errors.report('process:uncaughtException', {
+ source: err,
+ origin,
+ trace: true,
+ alarm: true,
+ });
+ });
+
+ });
+
+ process.on('unhandledRejection', async (reason, promise) => {
+ await Context.allow_fallback(async () => {
+ errors.report('process:unhandledRejection', {
+ source: reason,
+ promise,
+ trace: true,
+ alarm: true,
+ });
+ });
+ });
+ }
+}
+
+module.exports = {
+ ProcessEventService,
+};
diff --git a/packages/backend/src/services/runtime-analysis/ServerHealthService.js b/packages/backend/src/services/runtime-analysis/ServerHealthService.js
new file mode 100644
index 00000000..71fcf644
--- /dev/null
+++ b/packages/backend/src/services/runtime-analysis/ServerHealthService.js
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("../BaseService");
+const { SECOND } = require("../../util/time");
+const { parse_meminfo } = require("../../util/linux");
+const { asyncSafeSetInterval } = require("../../util/promise");
+
+class ServerHealthService extends BaseService {
+ static MODULES = {
+ fs: require('fs'),
+ }
+ async _init () {
+ const ram_poll_interval = 10 * SECOND;
+
+ /*
+ There's an interesting thread here:
+ https://github.com/nodejs/node/issues/23892
+
+ It's a discussion about whether to report "free" or "available" memory
+ in `os.freemem()`. There was no clear consensus in the discussion,
+ and then libuv was changed to report "available" memory instead.
+
+ I've elected not to use `os.freemem()` here and instead read
+ `/proc/meminfo` directly.
+ */
+
+
+ const min_free_KiB = 1024 * 1024; // 1 GiB
+ const min_available_KiB = 1024 * 1024 * 2; // 2 GiB
+
+ const svc_alarm = this.services.get('alarm');
+
+ this.stats_ = {};
+
+ // Disable if we're not on Linux
+ if ( process.platform !== 'linux' ) {
+ return;
+ }
+
+ asyncSafeSetInterval(async () => {
+ const meminfo_text = await this.modules.fs.promises.readFile(
+ '/proc/meminfo', 'utf8'
+ );
+ const meminfo = parse_meminfo(meminfo_text);
+ const alarm_fields = {
+ mem_free: meminfo.MemFree,
+ mem_available: meminfo.MemAvailable,
+ mem_total: meminfo.MemTotal,
+ };
+
+ Object.assign(this.stats_, alarm_fields);
+
+ if ( meminfo.MemAvailable < min_available_KiB ) {
+ svc_alarm.create('low-available-memory', 'Low available memory', alarm_fields);
+ }
+ }, ram_poll_interval, null,{
+ onBehindSchedule: (drift) => {
+ svc_alarm.create(
+ 'ram-usage-poll-behind-schedule',
+ 'RAM usage poll is behind schedule',
+ { drift }
+ );
+ }
+ });
+ }
+
+ async get_stats () {
+ return { ...this.stats_ };
+ }
+}
+
+module.exports = { ServerHealthService };
diff --git a/packages/backend/src/services/sla/MonthlyUsageService.js b/packages/backend/src/services/sla/MonthlyUsageService.js
new file mode 100644
index 00000000..2a8ab7c9
--- /dev/null
+++ b/packages/backend/src/services/sla/MonthlyUsageService.js
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("../BaseService");
+const { UserActorType, AppUnderUserActorType } = require("../auth/Actor");
+const { DB_WRITE } = require("../database/consts");
+
+class MonthlyUsageService extends BaseService {
+ async _init () {
+ this.db = this.services.get('database').get(DB_WRITE, 'usage');
+ }
+
+ async increment (actor, key, extra) {
+ key = `${actor.uid}:${key}`;
+
+ const year = new Date().getUTCFullYear();
+ // months are zero-indexed by getUTCMonth, which could be confusing
+ const month = new Date().getUTCMonth() + 1;
+
+ const maybe_app_id = actor.type.app?.id;
+
+ // UPSERT increment count
+ await this.db.write(
+ 'INSERT INTO `service_usage_monthly` (`year`, `month`, `key`, `count`, `user_id`, `app_id`, `extra`) ' +
+ 'VALUES (?, ?, ?, 1, ?, ?, ?) ' +
+ 'ON DUPLICATE KEY UPDATE `count` = `count` + 1',
+ [
+ year, month, key,
+ actor.type.user?.id || null,
+ maybe_app_id || null,
+ JSON.stringify(extra)
+ ]
+ );
+ }
+
+ async check (actor, specifiers) {
+ if ( actor.type instanceof UserActorType ) {
+ return await this._user_check(actor, specifiers);
+ }
+
+ if ( actor.type instanceof AppUnderUserActorType ) {
+ return await this._app_under_user_check(actor, specifiers);
+ }
+
+ }
+
+ async _user_check (actor, specifiers) {
+ const year = new Date().getUTCFullYear();
+ // months are zero-indexed by getUTCMonth, which could be confusing
+ const month = new Date().getUTCMonth() + 1;
+
+ const rows = await this.db.read(
+ 'SELECT SUM(`count`) AS sum FROM `service_usage_monthly` ' +
+ 'WHERE `year` = ? AND `month` = ? AND `user_id` = ? ' +
+ 'AND JSON_CONTAINS(`extra`, ?)',
+ [
+ year, month, actor.type.user.id,
+ JSON.stringify(specifiers),
+ ]
+ );
+
+ return rows[0]?.sum || 0;
+ }
+
+ async _app_under_user_check (actor, specifiers) {
+ const year = new Date().getUTCFullYear();
+ // months are zero-indexed by getUTCMonth, which could be confusing
+ const month = new Date().getUTCMonth() + 1;
+
+ const specifier_entries = Object.entries(specifiers);
+
+ // SELECT count
+ const rows = await this.db.read(
+ 'SELECT `count` FROM `service_usage_monthly` ' +
+ 'WHERE `year` = ? AND `month` = ? AND `user_id` = ? ' +
+ 'AND `app_id` = ? ' +
+ 'AND JSON_CONTAINS(`extra`, ?)',
+ [
+ year, month, actor.type.user.id,
+ actor.type.app.id,
+ specifiers,
+ ]
+ );
+
+ return rows[0]?.count || 0;
+ }
+}
+
+module.exports = {
+ MonthlyUsageService,
+};
diff --git a/packages/backend/src/services/sla/RateLimitService.js b/packages/backend/src/services/sla/RateLimitService.js
new file mode 100644
index 00000000..cec4735f
--- /dev/null
+++ b/packages/backend/src/services/sla/RateLimitService.js
@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+const APIError = require("../../api/APIError");
+const { Context } = require("../../util/context");
+const BaseService = require("../BaseService");
+const { SyncTrait } = require("../../traits/SyncTrait");
+const { DB_WRITE } = require("../database/consts");
+
+const ts_to_sql = (ts) => Math.floor(ts / 1000);
+const ts_fr_sql = (ts) => ts * 1000;
+
+class RateLimitService extends BaseService {
+ static MODULES = {
+ kv: globalThis.kv,
+ }
+
+ static TRAITS = [
+ new SyncTrait([
+ 'check_and_increment',
+ ]),
+ ]
+
+ async _init () {
+ this.db = this.services.get('database').get(DB_WRITE, 'rate-limit');
+ }
+
+ async check_and_increment (key, max, period, options = {}) {
+ const { kv } = this.modules;
+ const consumer_id = this._get_consumer_id();
+ const method_name = key;
+ key = `${consumer_id}:${key}`
+ const kvkey = `rate-limit:${key}`;
+ const dbkey = options.global ? key : `${this.global_config.server_id}:${key}`;
+
+ // Fixed window counter strategy (see devlog 2023-11-21)
+ let window_start = kv.get(`${kvkey}:window_start`) ?? 0;
+ if ( window_start === 0 ) {
+ // Try database
+ const rows = await this.db.read(
+ 'SELECT * FROM `rl_usage_fixed_window` WHERE `key` = ?',
+ [dbkey]
+ );
+
+ if ( rows.length !== 0 ) {
+ const row = rows[0];
+ window_start = ts_fr_sql(row.window_start);
+ const count = row.count;
+
+ console.log(
+ 'set window_start and count from DATABASE',
+ { window_start, count }
+ );
+
+ kv.set(`${kvkey}:window_start`, window_start);
+ kv.set(`${kvkey}:count`, count);
+ }
+ }
+
+ if ( window_start === 0 ) {
+ window_start = Date.now();
+ kv.set(`${kvkey}:window_start`, window_start);
+ kv.set(`${kvkey}:count`, 0);
+
+ await this.db.write(
+ 'INSERT INTO `rl_usage_fixed_window` (`key`, `window_start`, `count`) VALUES (?, ?, ?)',
+ [dbkey, ts_to_sql(window_start), 0]
+ );
+
+ console.log(
+ 'CREATE window_start and count',
+ { window_start, count: 0 }
+ );
+ }
+
+ console.log(
+ 'DEBUGGING COMPARISON',
+ { window_start, period, now: Date.now() }
+ );
+
+ if ( window_start + period < Date.now() ) {
+ window_start = Date.now();
+ kv.set(`${kvkey}:window_start`, window_start);
+ kv.set(`${kvkey}:count`, 0);
+
+ console.log(
+ 'REFRESH window_start and count',
+ { window_start, count: 0 }
+ );
+
+ await this.db.write(
+ 'UPDATE `rl_usage_fixed_window` SET `window_start` = ?, `count` = ? WHERE `key` = ?',
+ [ts_to_sql(window_start), 0, dbkey]
+ );
+ }
+
+ const current = kv.get(`${kvkey}:count`) ?? 0;
+ if ( current >= max ) {
+ throw APIError.create('rate_limit_exceeded', null, {
+ method_name,
+ rate_limit: { max, period }
+ });
+ }
+
+ kv.incr(`${kvkey}:count`);
+ await this.db.write(
+ 'UPDATE `rl_usage_fixed_window` SET `count` = `count` + 1 WHERE `key` = ?',
+ [dbkey]
+ );
+ }
+
+ _get_consumer_id () {
+ const context = Context.get();
+ const user = context.get('user');
+ return user ? `user:${user.id}` : 'missing';
+ }
+}
+
+module.exports = {
+ RateLimitService,
+};
diff --git a/packages/backend/src/services/sla/SLAService.js b/packages/backend/src/services/sla/SLAService.js
new file mode 100644
index 00000000..9a49547a
--- /dev/null
+++ b/packages/backend/src/services/sla/SLAService.js
@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require("../BaseService");
+
+/**
+ * SLAService is responsible for getting the appropriate SLA for a given
+ * driver or service endpoint, including limits with respect to the actor
+ * and server-wide limits.
+ */
+class SLAService extends BaseService {
+ async _construct () {
+ // I'm not putting this in config for now until we have checks
+ // for production configuration. - EAD
+ this.hardcoded_limits = {
+ system: {
+ 'driver:impl:public-helloworld:greet': {
+ rate_limit: {
+ max: 1000,
+ period: 30000,
+ },
+ },
+ 'driver:impl:public-aws-textract:recognize': {
+ rate_limit: {
+ max: 10,
+ period: 30000,
+ },
+ monthly_limit: 80 * 1000,
+ },
+ },
+ // app_default: {
+ // 'driver:impl:public-aws-textract:recognize': {
+ // rate_limit: {
+ // max: 40,
+ // period: 30000,
+ // },
+ // monthly_limit: 1000,
+ // },
+ // 'driver:impl:public-openai-chat-completion:complete': {
+ // rate_limit: {
+ // max: 30,
+ // period: 1000 * 60 * 60,
+ // },
+ // monthly_limit: 600,
+ // },
+ // 'driver:impl:public-openai-image-generation:generate': {
+ // rate_limit: {
+ // max: 30,
+ // period: 1000 * 60 * 60,
+ // },
+ // monthly_limit: 10000,
+ // },
+ // },
+ user_unverified: {
+ 'driver:impl:public-aws-textract:recognize': {
+ rate_limit: {
+ max: 40,
+ period: 30000,
+ },
+ monthly_limit: 20,
+ },
+ 'driver:impl:public-openai-chat-completion:complete': {
+ rate_limit: {
+ max: 40,
+ period: 30000,
+ },
+ monthly_limit: 100,
+ },
+ 'driver:impl:public-openai-image-generation:generate': {
+ rate_limit: {
+ max: 40,
+ period: 30000,
+ },
+ monthly_limit: 4,
+ },
+ },
+ user_verified: {
+ 'driver:impl:public-aws-textract:recognize': {
+ rate_limit: {
+ max: 40,
+ period: 30000,
+ },
+ monthly_limit: 100,
+ },
+ 'driver:impl:public-openai-chat-completion:complete': {
+ rate_limit: {
+ max: 40,
+ period: 30000,
+ },
+ monthly_limit: 1000,
+ },
+ 'driver:impl:public-openai-image-generation:generate': {
+ rate_limit: {
+ max: 40,
+ period: 30000,
+ },
+ monthly_limit: 5,
+ },
+ }
+ };
+ }
+
+ get (category, key) {
+ return this.hardcoded_limits[category]?.[key];
+ }
+}
+
+module.exports = {
+ SLAService,
+};
diff --git a/packages/backend/src/services/thumbnails/HTTPThumbnailService.js b/packages/backend/src/services/thumbnails/HTTPThumbnailService.js
new file mode 100644
index 00000000..309b106d
--- /dev/null
+++ b/packages/backend/src/services/thumbnails/HTTPThumbnailService.js
@@ -0,0 +1,291 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+// TODO: If an RPC protocol is ever used this service can be replaced
+// with a more general RPCService and a model.
+
+const axios = require('axios');
+
+const { TeePromise } = require("../../util/promise");
+const { AdvancedBase } = require('puter-js-common');
+const FormData = require("form-data");
+const { stream_to_the_void, buffer_to_stream } = require('../../util/streamutil');
+const BaseService = require('../BaseService');
+
+class ThumbnailOperation extends TeePromise {
+ // static MAX_RECYCLE_COUNT = 5*3;
+ static MAX_RECYCLE_COUNT = 3;
+ constructor (file) {
+ super();
+ this.file = file;
+ this.recycle_count = 0;
+ }
+
+ recycle () {
+ this.recycle_count++;
+
+ if ( this.recycle_count > this.constructor.MAX_RECYCLE_COUNT ) {
+ this.resolve(undefined);
+ return false;
+ }
+
+ return true;
+ }
+}
+
+class HTTPThumbnailService extends BaseService {
+ static STATUS_IDLE = {};
+ static STATUS_RUNNING = {};
+
+ static LIMIT = 400 * 1024 * 1024;
+
+ static MODULES = {
+ setTimeout,
+ axios,
+ };
+
+ static SUPPORTED_MIMETYPES = [
+ 'audio/ogg',
+ 'audio/wave',
+ 'audio/mpeg',
+ 'application/ogg',
+ 'application/pdf',
+ // 'image/bmp',
+ 'image/gif',
+ 'image/jpeg',
+ 'image/jpg',
+ 'image/png',
+ // 'image/tiff',
+ 'image/webp',
+ 'video/avi',
+ 'video/x-msvideo',
+ 'video/msvideo',
+ 'video/flv',
+ 'video/x-flv',
+ 'video/mp4',
+ 'video/x-matroska',
+ 'video/quicktime',
+ 'video/webm',
+ ];
+
+ constructor (cons) {
+ const { services, my_config } = cons;
+ super(cons);
+
+ this.services = services;
+ this.log = services.get('log-service').create('thumbnail-service');
+ this.errors = services.get('error-service').create(this.log);
+ this.config = my_config;
+
+ this.queue = [];
+ this.status = this.constructor.STATUS_IDLE;
+
+ this.LIMIT = my_config?.limit ?? this.constructor.LIMIT;
+
+ if ( my_config?.query_supported_types !== false ) {
+ setInterval(() => {
+ this.query_supported_mime_types_();
+ }, 60 * 1000);
+ }
+ }
+
+ get host_ () {
+ return this.config.host || 'http://127.0.0.1:3101';
+ }
+
+ is_supported_mimetype (mimetype) {
+ return this.constructor.SUPPORTED_MIMETYPES.includes(mimetype);
+ }
+
+ is_supported_size (size) {
+ return size < this.LIMIT;
+ }
+
+ /**
+ *
+ * @param {object} file - An object describing the file in the same format
+ * as the file object created by multer. The necessary properties are
+ * `buffer`, `filename`, and `mimetype`.
+ */
+ async thumbify(file) {
+ const job = new ThumbnailOperation(file);
+ this.queue.push(job);
+ this.checkShouldExec_();
+ return await job;
+ }
+
+ checkShouldExec_ () {
+ if ( this.status !== this.constructor.STATUS_IDLE ) return;
+ if ( this.queue.length === 0 ) return;
+ this.exec_();
+ }
+
+ async exec_ () {
+ const { setTimeout } = this.modules;
+
+ this.status = this.constructor.STATUS_RUNNING;
+
+ const LIMIT = this.LIMIT;
+
+ // Grab up to 400MB worth of files to send to the thumbnail service.
+ // Resolve any jobs as undefined if they're over the limit.
+
+ let total_size = 0;
+ const queue = [];
+ while ( this.queue.length > 0 ) {
+ const job = this.queue[0];
+ const size = job.file.size;
+ if ( size > LIMIT ) {
+ job.resolve(undefined);
+ if ( job.file.stream ) stream_to_the_void(job.file.stream);
+ this.queue.shift();
+ continue;
+ }
+ if ( total_size + size > LIMIT ) break;
+ total_size += size;
+ queue.push(job);
+ this.queue.shift();
+ }
+
+ if ( queue.length === 0 ) {
+ this.status = this.constructor.STATUS_IDLE;
+ return;
+ }
+
+ try {
+ return await this.exec_0(queue);
+ } catch (err) {
+ await new Promise(resolve => setTimeout(resolve, 200));
+
+ // const new_queue = queue.filter(job => job.recycle());
+ // this.queue = new_queue.concat(this.queue);
+ this.queue = [];
+ for ( const job of queue ) {
+ if ( job.file.stream ) stream_to_the_void(job.file.stream);
+ job.resolve(undefined);
+ }
+
+ this.errors.report('thumbnails-exec', {
+ source: err,
+ trace: true,
+ alarm: true,
+ });
+ } finally {
+ this.status = this.constructor.STATUS_IDLE;
+ this.checkShouldExec_();
+ }
+ }
+
+ async exec_0 (queue) {
+ const { axios } = this.modules;
+
+ let expected = 0;
+
+ const form = new FormData();
+ for ( const job of queue ) {
+ expected++;
+ // const blob = new Blob([job.file.buffer], { type: job.file.mimetype });
+ // form.append('file', blob, job.file.filename);
+ const file_data = job.file.buffer ? (() => {
+ job.file.size = job.file.buffer.length;
+ return buffer_to_stream(job.file.buffer);
+ })() : job.file.stream;
+ // const file_data = job.file.buffer ?? job.file.stream;
+ console.log('INFORMATION ABOUT THIS FILE', {
+ file_has_a_buffer: !!job.file.buffer,
+ file_has_a_stream: !!job.file.stream,
+ file: job.file,
+ });
+ form.append('file', file_data, {
+ filename: job.file.name ?? job.file.originalname,
+ contentType: job.file.type ?? job.file.mimetype,
+ knownLength: job.file.size,
+ });
+ }
+
+ this.log.info('starting thumbnail request');
+ const resp = await axios.request(
+ {
+ method: 'post',
+ url: `${this.host_}/thumbify`,
+ data: form,
+ headers: {
+ 'Content-Type': 'multipart/form-data',
+ }
+ }
+ );
+ this.log.info('done thumbnail request');
+
+ if ( resp.status !== 200 ) {
+ this.log.error('Thumbnail service returned non-200 status');
+ throw new Error('Thumbnail service returned non-200 status');
+ }
+
+ const results = resp.data;
+
+ this.log.noticeme('response?', { resp });
+ this.log.noticeme('data?', { data: resp.data });
+
+ if ( results.length !== queue.length ) {
+ this.log.error('Thumbnail service returned wrong number of results');
+ throw new Error('Thumbnail service returned wrong number of results');
+ }
+
+ for ( let i = 0 ; i < queue.length ; i++ ) {
+ const result = results[i];
+ const job = queue[i];
+
+ this.log.noticeme('result?', { result });
+ job.resolve(
+ result.encoded
+ && `data:image/png;base64,${result.encoded}`
+ );
+ }
+ }
+
+ async query_supported_mime_types_() {
+ const resp = await axios.request(
+ {
+ method: 'get',
+ url: `${this.host_}/supported`,
+ }
+ );
+
+ const data = resp.data;
+
+ if ( ! Array.isArray(data) ) {
+ this.log.error('Thumbnail service returned invalid data');
+ return;
+ }
+
+ const mime_set = {};
+
+ for ( const entry of data ) {
+ mime_set[entry.StandardMIMEType] = true;
+ for ( const mime of entry.MIMETypes ) {
+ mime_set[mime] = true;
+ }
+ }
+
+ this.constructor.SUPPORTED_MIMETYPES = Object.keys(mime_set);
+ }
+}
+
+module.exports = {
+ HTTPThumbnailService,
+};
diff --git a/packages/backend/src/services/thumbnails/NAPIThumbnailService.js b/packages/backend/src/services/thumbnails/NAPIThumbnailService.js
new file mode 100644
index 00000000..0b76073b
--- /dev/null
+++ b/packages/backend/src/services/thumbnails/NAPIThumbnailService.js
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const BaseService = require('../BaseService');
+
+class NAPIThumbnailService extends BaseService {
+ static LIMIT = 400 * 1024 * 1024;
+ static SUPPORTED_MIMETYPES = [
+ "image/jpeg",
+ "image/png",
+ "image/webp",
+ "image/gif",
+ "image/avif",
+ "image/tiff",
+ "image/svg+xml"
+ ];
+
+ static MODULES = {
+ sharp: () => require('sharp'),
+ };
+
+ is_supported_mimetype (mimetype) {
+ return this.constructor.SUPPORTED_MIMETYPES.includes(mimetype);
+ }
+ is_supported_size (size) {
+ return size <= this.constructor.LIMIT;
+ }
+ async thumbify (file) {
+ const transformer = await this.modules.sharp()()
+ .resize(128)
+ .png();
+ file.stream.pipe(transformer);
+ const buffer = await transformer.toBuffer();
+ // .toBuffer();
+ const base64 = buffer.toString('base64');
+ return `data:image/png;base64,${base64}`;
+ }
+}
+
+module.exports = {
+ NAPIThumbnailService,
+};
diff --git a/packages/backend/src/services/thumbnails/PureJSThumbnailService.js b/packages/backend/src/services/thumbnails/PureJSThumbnailService.js
new file mode 100644
index 00000000..ad98f558
--- /dev/null
+++ b/packages/backend/src/services/thumbnails/PureJSThumbnailService.js
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const Jimp = require('jimp');
+const BaseService = require('../BaseService');
+const { stream_to_buffer } = require('../../util/streamutil');
+
+class PureJSThumbnailService extends BaseService {
+ static DESCRIPTION = `
+ This thumbnail service doesn't depend on any low-level compiled
+ libraries. It is CPU-intensive, so it's not ideal for production
+ deployments, but it's great for development and testing.
+ `;
+
+ static LIMIT = 400 * 1024 * 1024;
+ static SUPPORTED_MIMETYPES = [
+ "image/jpeg",
+ "image/png",
+ "image/bmp",
+ "image/tiff",
+ "image/gif"
+ ]
+
+ static MODULES = {
+ jimp: require('jimp'),
+ };
+
+ is_supported_mimetype (mimetype) {
+ return this.constructor.SUPPORTED_MIMETYPES.includes(mimetype);
+ }
+ is_supported_size (size) {
+ return size <= this.constructor.LIMIT;
+ }
+
+ async thumbify (file) {
+ const buffer = await stream_to_buffer(file.stream);
+ const image = await Jimp.read(buffer);
+ image.resize(128, 128);
+ const base64 = await image.getBase64Async(Jimp.MIME_PNG);
+ return base64;
+ }
+}
+
+module.exports = {
+ PureJSThumbnailService,
+};
diff --git a/packages/backend/src/socketio.js b/packages/backend/src/socketio.js
new file mode 100644
index 00000000..8b86a419
--- /dev/null
+++ b/packages/backend/src/socketio.js
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+let io;
+module.exports = {
+ init: function(server) {
+ // start socket.io server and cache io value
+ io = require('socket.io')(server, {
+ cors: {
+ origin: '*',
+ }
+ });
+ // io.origins('*:*');
+ return io;
+ },
+ getio: function() {
+ // return previously cached value
+ if (!io) {
+ throw new Error("must call .init(server) before you can call .getio()");
+ }
+ return io;
+ }
+}
\ No newline at end of file
diff --git a/packages/backend/src/temp/puter_page_loader.js b/packages/backend/src/temp/puter_page_loader.js
new file mode 100644
index 00000000..af352101
--- /dev/null
+++ b/packages/backend/src/temp/puter_page_loader.js
@@ -0,0 +1,161 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const {encode} = require('html-entities');
+const path_ = require('path');
+const fs_ = require('fs');
+
+const generate_puter_page_html = ({
+ manifest,
+ gui_path,
+
+ app_origin,
+ api_origin,
+
+ meta,
+
+ gui_params,
+}) => {
+ const e = encode;
+
+ const {
+ title,
+ description,
+ short_description,
+ company,
+ canonical_url,
+ } = meta;
+
+ gui_params = {
+ ...meta,
+ ...gui_params,
+ app_origin,
+ api_origin,
+ gui_origin: app_origin,
+ };
+
+ return `
+
+
+
+ ${e(title)}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${
+ (manifest?.css_paths
+ ? manifest.css_paths.map(path => ` \n`)
+ : []).join('')
+ }
+
+
+
+
+
+ ${
+ (manifest?.lib_paths
+ ? manifest.lib_paths.map(path => `\n`)
+ : []).join('')
+ }
+
+
+
+ ${
+ (manifest?.js_paths
+ ? manifest.js_paths.map(path => `\n`)
+ : []).join('')
+ }
+
+
+
+
+
+
+
+`;
+};
+
+module.exports = {
+ generate_puter_page_html,
+};
diff --git a/packages/backend/src/traits/AssignableMethodsTrait.js b/packages/backend/src/traits/AssignableMethodsTrait.js
new file mode 100644
index 00000000..0730be46
--- /dev/null
+++ b/packages/backend/src/traits/AssignableMethodsTrait.js
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class AssignableMethodsTrait {
+ install_in_instance (instance) {
+ const methods = instance._get_merged_static_object('METHODS');
+
+ for ( const k in methods ) {
+ instance[k] = methods[k];
+ }
+ }
+}
+
+module.exports = {
+ AssignableMethodsTrait
+};
diff --git a/packages/backend/src/traits/AsyncProviderTrait.js b/packages/backend/src/traits/AsyncProviderTrait.js
new file mode 100644
index 00000000..280ac6ca
--- /dev/null
+++ b/packages/backend/src/traits/AsyncProviderTrait.js
@@ -0,0 +1,120 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class AsyncProviderTrait {
+ install_in_instance (instance) {
+ instance.valueListeners_ = {};
+ instance.valueFactories_ = {};
+ instance.values_ = {};
+ instance.rejections_ = {};
+
+ instance.provideValue = AsyncProviderTrait.prototype.provideValue;
+ instance.rejectValue = AsyncProviderTrait.prototype.rejectValue;
+ instance.awaitValue = AsyncProviderTrait.prototype.awaitValue;
+ instance.onValue = AsyncProviderTrait.prototype.onValue;
+ instance.setFactory = AsyncProviderTrait.prototype.setFactory;
+ }
+
+ provideValue (key, value) {
+ this.values_[key] = value;
+
+ let listeners = this.valueListeners_[key];
+ if ( ! listeners ) return;
+
+ delete this.valueListeners_[key];
+
+ for ( let listener of listeners ) {
+ if ( Array.isArray(listener) ) listener = listener[0];
+ listener(value);
+ }
+ }
+
+ rejectValue (key, err) {
+ this.rejections_[key] = err;
+
+ let listeners = this.valueListeners_[key];
+ if ( ! listeners ) return;
+
+ delete this.valueListeners_[key];
+
+ for ( let listener of listeners ) {
+ if ( ! Array.isArray(listener) ) continue;
+ if ( ! listener[1] ) continue;
+ listener = listener[1];
+
+ listener(err);
+ }
+ }
+
+ awaitValue (key) {
+ return new Promise ((rslv, rjct) => {
+ this.onValue(key, rslv, rjct);
+ });
+ }
+
+ onValue (key, fn, rjct) {
+ if ( this.values_[key] ) {
+ fn(this.values_[key]);
+ return;
+ }
+
+ if ( this.rejections_[key] ) {
+ if ( rjct ) {
+ rjct(this.rejections_[key]);
+ } else throw this.rejections_[key];
+ return;
+ }
+
+ if ( ! this.valueListeners_[key] ) {
+ this.valueListeners_[key] = [];
+ }
+ this.valueListeners_[key].push([fn, rjct]);
+
+ if ( this.valueFactories_[key] ) {
+ const fn = this.valueFactories_[key];
+ delete this.valueFactories_[key];
+ (async () => {
+ try {
+ const value = await fn();
+ this.provideValue(key, value);
+ } catch (e) {
+ this.rejectValue(key, e);
+ }
+ })();
+ }
+ }
+
+ async setFactory (key, factoryFn) {
+ if ( this.valueListeners_[key] ) {
+ let v;
+ try {
+ v = await factoryFn();
+ } catch (e) {
+ this.rejectValue(key, e);
+ }
+ this.provideValue(key, v);
+ return;
+ }
+
+ this.valueFactories_[key] = factoryFn;
+ }
+}
+
+module.exports = {
+ AsyncProviderTrait
+};
\ No newline at end of file
diff --git a/packages/backend/src/traits/ContextAwareTrait.js b/packages/backend/src/traits/ContextAwareTrait.js
new file mode 100644
index 00000000..e95cfc6e
--- /dev/null
+++ b/packages/backend/src/traits/ContextAwareTrait.js
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Context } = require("../util/context");
+
+class ContextAwareTrait {
+ install_in_instance (instance) {
+ instance.context = Context.get();
+ instance.x = instance.context;
+ }
+}
+
+module.exports = {
+ ContextAwareTrait,
+};
diff --git a/packages/backend/src/traits/OtelTrait.js b/packages/backend/src/traits/OtelTrait.js
new file mode 100644
index 00000000..422281e2
--- /dev/null
+++ b/packages/backend/src/traits/OtelTrait.js
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Context } = require("../util/context");
+
+class OtelTrait {
+ constructor (method_include_list) {
+ this.method_include_list = method_include_list;
+ }
+ install_in_instance (instance) {
+ for ( const method_name of this.method_include_list ) {
+ const original_method = instance[method_name];
+ instance[method_name] = async (...args) => {
+ const context = Context.get();
+ // This happens when internal services call, such as PuterVersionService
+ if ( ! context ) return;
+
+ const class_name = instance.constructor.name;
+
+ const tracer = context.get('services').get('traceService').tracer;
+ let result;
+ await tracer.startActiveSpan(`${class_name}:${method_name}`, async span => {
+ result = await original_method.call(instance, ...args);
+ span.end();
+ });
+ return result;
+ }
+ }
+ }
+}
+
+class SyncOtelTrait {
+ constructor (method_include_list) {
+ this.method_include_list = method_include_list;
+ }
+ install_in_instance (instance) {
+ for ( const method_name of this.method_include_list ) {
+ const original_method = instance[method_name];
+ instance[method_name] = (...args) => {
+ const context = Context.get();
+ if ( ! context ) {
+ throw new Error('missing context');
+ }
+
+ const class_name = instance.constructor.name;
+
+ const tracer = context.get('services').get('traceService').tracer;
+ let result;
+ tracer.startActiveSpan(`${class_name}:${method_name}`, async span => {
+ result = original_method.call(instance, ...args);
+ span.end();
+ });
+ return result;
+ }
+ }
+ }
+}
+
+module.exports = {
+ OtelTrait
+};
diff --git a/packages/backend/src/traits/SyncTrait.js b/packages/backend/src/traits/SyncTrait.js
new file mode 100644
index 00000000..7257ef55
--- /dev/null
+++ b/packages/backend/src/traits/SyncTrait.js
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { Lock } = require("../util/promise");
+
+class SyncTrait {
+ constructor (method_include_list) {
+ this.method_include_list = method_include_list;
+ }
+
+ install_in_instance (instance) {
+ for ( const method_name of this.method_include_list ) {
+ const original_method = instance[method_name];
+ const lock = new Lock();
+ instance[method_name] = async (...args) => {
+ return await lock.acquire(async () => {
+ return await original_method.call(instance, ...args);
+ });
+ }
+ }
+ }
+}
+
+module.exports = {
+ SyncTrait,
+};
diff --git a/packages/backend/src/traits/WeakConstructorTrait.js b/packages/backend/src/traits/WeakConstructorTrait.js
new file mode 100644
index 00000000..8b3b653d
--- /dev/null
+++ b/packages/backend/src/traits/WeakConstructorTrait.js
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class WeakConstructorTrait {
+ install_in_instance(instance, { parameters }) {
+ for ( const key in parameters ) {
+ instance[key] = parameters[key];
+ }
+ }
+}
+
+module.exports = {
+ WeakConstructorTrait,
+};
diff --git a/packages/backend/src/user-mig.js b/packages/backend/src/user-mig.js
new file mode 100644
index 00000000..0b600a27
--- /dev/null
+++ b/packages/backend/src/user-mig.js
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+"use strict"
+const db = require('./db/mysql.js')
+const { mkdir } = require('./helpers');
+
+(async function() {
+ // get users
+ const [users] = await db.promise().execute( `SELECT * FROM user`);
+
+ // for each user ...
+ for(let i=0; i .
+ */
+class ConsoleLogManager {
+ static instance_;
+
+ static getInstance () {
+ if ( this.instance_ ) return this.instance_;
+ return this.instance_ = new ConsoleLogManager();
+ }
+
+ static CONSOLE_METHODS = [
+ 'log', 'error', 'warn',
+ ];
+
+ static PROXY_METHOD = function (method, ...args) {
+ const decorators = this.get_log_decorators_(method);
+
+ // TODO: Add this feature later
+ // const pre_listeners = self.get_log_pre_listeners_(method);
+ // const post_listeners = self.get_log_post_listeners_(method);
+
+ const replace = (...newargs) => {
+ args = newargs;
+ }
+ for ( const dec of decorators ) {
+ dec({
+ manager: this,
+ replace
+ }, ...args);
+ }
+
+ this.__original_methods[method](...args);
+
+ const post_hooks = this.get_post_hooks_(method);
+ for ( const fn of post_hooks ) {
+ fn();
+ }
+ }
+
+ get_log_decorators_ (method) {
+ return this.__log_decorators[method];
+ }
+
+ get_post_hooks_ (method) {
+ return this.__log_hooks_post[method];
+ }
+
+ constructor () {
+ const THIS = this.constructor;
+ this.__original_console = console;
+ this.__original_methods = {};
+ for ( const k of THIS.CONSOLE_METHODS ) {
+ this.__original_methods[k] = console[k];
+ }
+ this.__proxy_methods = {};
+ this.__log_decorators = {};
+ this.__log_hooks_post = {};
+
+ // TODO: Add this feature later
+ // this.__log_pre_listeners = {};
+ // this.__log_post_listeners = {};
+ }
+
+ initialize_proxy_methods (methods) {
+ const THIS = this.constructor;
+ methods = methods || THIS.CONSOLE_METHODS;
+ for ( const k of methods ) {
+ this.__proxy_methods[k] = THIS.PROXY_METHOD.bind(this, k);
+ console[k] = this.__proxy_methods[k];
+ this.__log_decorators[k] = [];
+ this.__log_hooks_post[k] = [];
+ }
+ }
+
+ decorate (method, dec_fn) {
+ this.__log_decorators[method] = dec_fn;
+ }
+
+ decorate_all (dec_fn) {
+ const THIS = this.constructor;
+ for ( const k of THIS.CONSOLE_METHODS ) {
+ this.__log_decorators[k].push(dec_fn);
+ }
+ }
+
+ post_all (post_fn) {
+ const THIS = this.constructor;
+ for ( const k of THIS.CONSOLE_METHODS ) {
+ this.__log_hooks_post[k].push(post_fn);
+ }
+ }
+
+ log_raw (method, ...args) {
+ this.__original_methods[method](...args);
+ }
+}
+
+module.exports = {
+ consoleLogManager: ConsoleLogManager.getInstance()
+};
diff --git a/packages/backend/src/util/context.js b/packages/backend/src/util/context.js
new file mode 100644
index 00000000..4e222caf
--- /dev/null
+++ b/packages/backend/src/util/context.js
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AsyncLocalStorage } = require('async_hooks');
+const config = require('../config');
+const { generate_random_code } = require('./identifier');
+
+class Context {
+ static USE_NAME_FALLBACK = {};
+ static next_name_ = 0;
+ static other_next_names_ = {};
+
+ static contextAsyncLocalStorage = new AsyncLocalStorage();
+ static __last_context_key = 0;
+ static make_context_key (opt_human_readable) {
+ let k = `_:${++this.__last_context_key}`;
+ if ( opt_human_readable ) {
+ k += `:${opt_human_readable}`;
+ }
+ return k;
+ }
+ static create (values, opt_name) {
+ return new Context(values, undefined, opt_name);
+ }
+ static get (k, { allow_fallback } = {}) {
+ let x = this.contextAsyncLocalStorage.getStore()?.get('context');
+ if ( ! x ) {
+ if ( config.env === 'dev' && ! allow_fallback ) {
+ throw new Error(
+ 'FAILED TO GET THE CORRECT CONTEXT'
+ );
+ }
+
+ // x = globalThis.root_context ?? this.create({});
+ x = this.root.sub({}, this.USE_NAME_FALLBACK);
+ }
+ if ( x && k ) return x.get(k);
+ return x;
+ }
+ static set (k, v) {
+ const x = this.contextAsyncLocalStorage.getStore()?.get('context');
+ if ( x ) return x.set(k, v);
+ }
+ static root = new Context({}, undefined, 'root');
+ static describe () {
+ return this.get().describe();
+ }
+ get (k) {
+ return this.values_[k];
+ }
+ set (k, v) {
+ this.values_[k] = v;
+ }
+ sub (values, opt_name) {
+ return new Context(values, this, opt_name);
+ }
+ get values () {
+ return this.values_;
+ }
+ constructor (imm_values, opt_parent, opt_name) {
+ const values = { ...imm_values };
+ imm_values = null;
+
+ opt_parent = opt_parent || Context.root;
+
+ this.name = (() => {
+ if ( opt_name === this.constructor.USE_NAME_FALLBACK ) {
+ opt_name = 'F';
+ }
+ if ( opt_name ) {
+ const name_numbers = this.constructor.other_next_names_;
+ if ( ! name_numbers.hasOwnProperty(opt_name) ) {
+ name_numbers[opt_name] = 0;
+ }
+ const num = ++name_numbers[opt_name];
+ return `{${opt_name}:${num}}`;
+ }
+ return `${++this.constructor.next_name_}`;
+ })();
+ this.parent_ = opt_parent;
+
+ if ( opt_parent ) {
+ values.__proto__ = opt_parent.values_;
+ for ( const k in values ) {
+ const parent_val = opt_parent.values_[k];
+ if ( parent_val instanceof Context ) {
+ if ( ! (values[k] instanceof Context) ) {
+ values[k] = parent_val.sub(values[k]);
+ }
+ }
+ }
+ }
+
+ this.values_ = values;
+ }
+ async arun (cb) {
+ const als = this.constructor.contextAsyncLocalStorage;
+ return await als.run(new Map(), async () => {
+ als.getStore().set('context', this);
+ return await cb();
+ });
+ }
+ abind (cb) {
+ const als = this.constructor.contextAsyncLocalStorage;
+ return async (...args) => {
+ return await this.arun(async () => {
+ return await cb(...args);
+ });
+ };
+ }
+
+ describe () {
+ return `Context(${this.describe_()})`;
+ }
+ describe_ () {
+ if ( ! this.parent_ ) return `[R]`;
+ return `${this.parent_.describe_()}->${this.name}`;
+ }
+
+ static async allow_fallback (cb) {
+ const x = this.get(undefined, { allow_fallback: true });
+ return await x.arun(async () => {
+ return await cb();
+ });
+ }
+}
+
+const uuidv4 = require('uuid').v4;
+
+class ContextExpressMiddleware {
+ constructor ({ parent }) {
+ this.parent_ = parent;
+ }
+ install (app) {
+ app.use(this.run.bind(this));
+ }
+ async run (req, res, next) {
+ return await this.parent_.sub({
+ req, res,
+ trace_request: uuidv4(),
+ }, 'req').arun(async () => {
+ const ctx = Context.get();
+ req.ctx = ctx;
+ res.locals.ctx = ctx;
+ next();
+ });
+ }
+}
+
+module.exports = {
+ Context,
+ ContextExpressMiddleware,
+};
diff --git a/packages/backend/src/util/datautil.js b/packages/backend/src/util/datautil.js
new file mode 100644
index 00000000..0ccfa7dc
--- /dev/null
+++ b/packages/backend/src/util/datautil.js
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+/**
+ * Stringify an object in such a way that objects with differing
+ * key orderings will still be considered equal.
+ * @param {*} obj
+ */
+const stringify_serializable_object = obj => {
+ if ( obj === undefined ) return '[undefined]';
+ if ( obj === null ) return '[null]';
+ if ( typeof obj === 'function' ) return '[function]';
+ if ( typeof obj !== 'object' ) return JSON.stringify(obj);
+
+ // ensure an error is thrown if the object is not serializable.
+ // (instead of failing with a stack overflow)
+ JSON.stringify(obj);
+
+ const keys = Object.keys(obj).sort();
+ const pairs = keys.map(key => {
+ const value = stringify_serializable_object(obj[key]);
+ const outer_json = JSON.stringify({ [key]: value });
+ return outer_json.slice(1, -1);
+ });
+
+ return '{' + pairs.join(',') + '}';
+};
+
+const hash_serializable_object = obj => {
+ const crypto = require('crypto');
+ const str = stringify_serializable_object(obj);
+ return crypto.createHash('sha1').update(str).digest('hex');
+};
+
+module.exports = {
+ stringify_serializable_object,
+ hash_serializable_object,
+};
diff --git a/packages/backend/src/util/errorutil.js b/packages/backend/src/util/errorutil.js
new file mode 100644
index 00000000..b61162f9
--- /dev/null
+++ b/packages/backend/src/util/errorutil.js
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const log_http_error = e => {
+ console.log('\x1B[31;1m' + e.message + '\x1B[0m');
+
+ console.log('HTTP Method: ', e.config.method.toUpperCase());
+ console.log('URL: ', e.config.url);
+
+ if (e.config.params) {
+ console.log('URL Parameters: ', e.config.params);
+ }
+
+ if (e.config.method.toLowerCase() === 'post' && e.config.data) {
+ console.log('Post body: ', e.config.data);
+ }
+
+ console.log('Request Headers: ', JSON.stringify(e.config.headers, null, 2));
+
+ if (e.response) {
+ console.log('Response Status: ', e.response.status);
+ console.log('Response Headers: ', JSON.stringify(e.response.headers, null, 2));
+ console.log('Response body: ', e.response.data);
+ }
+
+ console.log('\x1B[31;1m' + e.message + '\x1B[0m');
+};
+
+const better_error_printer = e => {
+ if ( e.request ) {
+ log_http_error(e);
+ return;
+ }
+
+ console.error(e);
+};
+
+/**
+ * This class is used to wrap an error when the error has
+ * already been sent to ErrorService. This prevents higher-level
+ * error handlers from sending it to ErrorService again.
+ */
+class ManagedError extends Error {
+ constructor (source, extra = {}) {
+ super(source?.message ?? source);
+ this.source = source;
+ this.name = `Managed(${source?.name ?? 'Error'})`;
+ this.extra = extra;
+ }
+}
+
+module.exports = {
+ ManagedError,
+ better_error_printer,
+
+ // We export CompositeError from 'composite-error' here
+ // in case we want to change the implementation later.
+ // i.e. it's under the MIT license so it would be easier
+ // to just copy the class to this file than maintain a fork.
+ CompositeError: require('composite-error'),
+};
diff --git a/packages/backend/src/util/files.js b/packages/backend/src/util/files.js
new file mode 100644
index 00000000..cdb50cfa
--- /dev/null
+++ b/packages/backend/src/util/files.js
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const fs = require('fs').promises;
+
+async function fallbackRead(...files) {
+ let lastError = null;
+ for (const file of files) {
+ try {
+ const data = await fs.readFile(file, 'utf8');
+ return data;
+ } catch (error) {
+ lastError = error;
+ }
+ }
+ throw lastError;
+}
+
+module.exports = {
+ fallbackRead
+};
diff --git a/packages/backend/src/util/fuzz.js b/packages/backend/src/util/fuzz.js
new file mode 100644
index 00000000..b7676378
--- /dev/null
+++ b/packages/backend/src/util/fuzz.js
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+// function fuzz_number(n) {
+// if (n === 0) return 0;
+
+// // let randomized = n + (Math.random() - 0.5) * n * 0.2;
+// let randomized = n;
+// let magnitude = Math.floor(Math.log10(randomized));
+// let factor = Math.pow(10, magnitude);
+// return Math.round(randomized / factor) * factor;
+// }
+
+function fuzz_number(num) {
+ // If the number is 0, then return 0
+ if (num === 0) return 0;
+
+ const magnitude = Math.floor(Math.log10(Math.abs(num)));
+
+ let significantFigures;
+
+ if (magnitude < 2) { // Numbers < 100
+ significantFigures = magnitude + 1;
+ } else if (magnitude < 5) { // Numbers < 100,000
+ significantFigures = 2;
+ } else { // Numbers >= 100,000
+ significantFigures = 3;
+ }
+
+ const factor = Math.pow(10, magnitude - significantFigures + 1);
+ return Math.round(num / factor) * factor;
+}
+
+// function fuzz_number(number) {
+// if (isNaN(number)) {
+// return 'Invalid number';
+// }
+
+// let formattedNumber;
+// if (number >= 1000000) {
+// // For millions, we want to show one decimal place
+// formattedNumber = (number / 1000000).toFixed(0) + 'm';
+// } else if (number >= 1000) {
+// // For thousands, we want to show one decimal place
+// formattedNumber = (number / 1000).toFixed(0) + 'k';
+// } else if (number >= 500) {
+// // For hundreds, we want to show no decimal places
+// formattedNumber = '500+';
+// } else if (number >= 100) {
+// // For hundreds, we want to show no decimal places
+// formattedNumber = '100+';
+// } else if (number >= 50) {
+// // For hundreds, we want to show no decimal places
+// formattedNumber = '50+';
+// } else if (number >= 10) {
+// // For hundreds, we want to show no decimal places
+// formattedNumber = '10+';
+// }
+// else {
+// // For numbers less than 10, we show the number as is.
+// formattedNumber = '1+';
+// }
+
+// // If the decimal place is 0 (e.g., 5.0k), we remove the decimal part (to have 5k instead)
+// formattedNumber = formattedNumber.replace(/\.0(?=[k|m])/, '');
+
+// // Append the plus sign for numbers 1000 and greater, denoting the number is 'this value or more'.
+// if (number >= 1000) {
+// formattedNumber += '+';
+// }
+
+// return formattedNumber;
+// }
+
+module.exports = {
+ fuzz_number
+};
diff --git a/packages/backend/src/util/gcutil.js b/packages/backend/src/util/gcutil.js
new file mode 100644
index 00000000..976a3b7e
--- /dev/null
+++ b/packages/backend/src/util/gcutil.js
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+/**
+ * gc_friendly_rslv is based on a hunch about how the garbage collector works.
+ */
+const NOOP = () => {};
+const gc_friendly_rslv = (rslv) => {
+ return (value) => {
+ rslv(value);
+ rslv = NOOP;
+ };
+};
+
+module.exports = {
+ NOOP,
+ gc_friendly_rslv,
+};
diff --git a/packages/backend/src/util/hl_types.js b/packages/backend/src/util/hl_types.js
new file mode 100644
index 00000000..6f0abd5a
--- /dev/null
+++ b/packages/backend/src/util/hl_types.js
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { quot } = require("./strutil");
+
+const hl_type_definitions = {
+ flag: {
+ fallback: false,
+ required_check: v => {
+ if ( v === undefined || v === '' ) {
+ return false;
+ }
+ return true;
+ },
+ adapt: (v) => {
+ if ( typeof v === 'string' ) {
+ if (
+ v === 'true' || v === '1' || v === 'yes'
+ ) return true;
+
+ if (
+ v === 'false' || v === '0' || v === 'no'
+ ) return false;
+
+ throw new Error(`could not adapt string to boolean: ${quot(v)}`);
+ }
+
+ if ( typeof v === 'boolean' ) {
+ return v;
+ }
+
+ throw new Error(`could not adapt value to boolean: ${quot(v)}`);
+ }
+ }
+};
+
+class HLTypeFacade {
+ static REQUIRED = {};
+ static convert (type, value, opt_default) {
+ const type_definition = hl_type_definitions[type];
+ const has_value = type_definition.required_check(value);
+ if ( ! has_value ) {
+ if ( opt_default === HLTypeFacade.REQUIRED ) {
+ throw new Error(`required value is missing`);
+ }
+ return opt_default ?? type_definition.fallback;
+ }
+ return type_definition.adapt(value);
+ }
+}
+
+module.exports = {
+ hl_type_definitions,
+ HLTypeFacade,
+ boolify: HLTypeFacade.convert.bind(HLTypeFacade, 'flag'),
+};
diff --git a/packages/backend/src/util/identifier.js b/packages/backend/src/util/identifier.js
new file mode 100644
index 00000000..d693ff6b
--- /dev/null
+++ b/packages/backend/src/util/identifier.js
@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const adjectives = [
+ 'amazing', 'ambitious', 'articulate', 'cool', 'bubbly', 'mindful', 'noble', 'savvy', 'serene',
+ 'sincere', 'sleek', 'sparkling', 'spectacular', 'splendid', 'spotless', 'stunning',
+ 'awesome', 'beaming', 'bold', 'brilliant', 'cheerful', 'modest', 'motivated',
+ 'friendly', 'fun', 'funny', 'generous', 'gifted', 'graceful', 'grateful',
+ 'passionate', 'patient', 'peaceful', 'perceptive', 'persistent',
+ 'helpful', 'sensible', 'loyal', 'honest', 'clever', 'capable',
+ 'calm', 'smart', 'genius', 'bright', 'charming', 'creative', 'diligent', 'elegant', 'fancy',
+ 'colorful', 'avid', 'active', 'gentle', 'happy', 'intelligent',
+ 'jolly', 'kind', 'lively', 'merry', 'nice', 'optimistic', 'polite',
+ 'quiet', 'relaxed', 'silly', 'witty', 'young',
+ 'strong', 'brave', 'agile', 'bold', 'confident', 'daring',
+ 'fearless', 'heroic', 'mighty', 'powerful', 'valiant', 'wise', 'wonderful', 'zealous',
+ 'warm', 'swift', 'neat', 'tidy', 'nifty', 'lucky', 'keen',
+ 'blue', 'red', 'aqua', 'green', 'orange', 'pink', 'purple', 'cyan', 'magenta', 'lime',
+ 'teal', 'lavender', 'beige', 'maroon', 'navy', 'olive', 'silver', 'gold', 'ivory',
+];
+
+const nouns = [
+ 'street', 'roof', 'floor', 'tv', 'idea', 'morning', 'game', 'wheel', 'bag', 'clock', 'pencil', 'pen',
+ 'magnet', 'chair', 'table', 'house', 'room', 'book', 'car', 'tree', 'candle', 'light', 'planet',
+ 'flower', 'bird', 'fish', 'sun', 'moon', 'star', 'cloud', 'rain', 'snow', 'wind', 'mountain',
+ 'river', 'lake', 'sea', 'ocean', 'island', 'bridge', 'road', 'train', 'plane', 'ship', 'bicycle',
+ 'circle', 'square', 'garden', 'harp', 'grass', 'forest', 'rock', 'cake', 'pie', 'cookie', 'candy',
+ 'butterfly', 'computer', 'phone', 'keyboard', 'mouse', 'cup', 'plate', 'glass', 'door',
+ 'window', 'key', 'wallet', 'pillow', 'bed', 'blanket', 'soap', 'towel', 'lamp', 'mirror',
+ 'camera', 'hat', 'shirt', 'pants', 'shoes', 'watch', 'ring',
+ 'necklace', 'ball', 'toy', 'doll', 'kite', 'balloon', 'guitar', 'violin', 'piano', 'drum',
+ 'trumpet', 'flute', 'viola', 'cello', 'harp', 'banjo', 'tuba',
+]
+
+const words = {
+ adjectives,
+ nouns,
+};
+
+const randomItem = (arr, random) => arr[Math.floor((random ?? Math.random)() * arr.length)];
+
+/**
+ * A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999).
+ * The result is returned as a string with components separated by the specified separator.
+ * It is useful when you need to create unique identifiers that are also human-friendly.
+ *
+ * @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided.
+ * @returns {string} A unique, human-friendly identifier.
+ *
+ * @example
+ *
+ * let identifier = window.generate_identifier();
+ * // identifier would be something like 'clever-idea-123'
+ *
+ */
+function generate_identifier(separator = '_', rng = Math.random){
+ // return a random combination of first_adj + noun + number (between 0 and 9999)
+ // e.g. clever-idea-123
+ return [
+ randomItem(adjectives, rng),
+ randomItem(nouns, rng),
+ Math.floor(rng() * 10000),
+ ].join(separator);
+}
+
+const HUMAN_READABLE_CASE_INSENSITIVE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
+
+function generate_random_code(n, {
+ rng = Math.random,
+ chars = HUMAN_READABLE_CASE_INSENSITIVE
+} = {}) {
+ let code = '';
+ for ( let i = 0 ; i < n ; i++ ) {
+ code += randomItem(chars, rng);
+ }
+ return code;
+}
+
+/**
+ *
+ * @param {*} n length of output code
+ * @param {*} mask - a string of characters to start with
+ * @param {*} value - a number to be converted to base-36 and put on the right
+ */
+function compose_code(mask, value) {
+ const right_str = value.toString(36);
+ let out_str = mask;
+ console.log('right_str', right_str);
+ console.log('out_str', out_str);
+ for ( let i = 0 ; i < right_str.length ; i++ ) {
+ out_str[out_str.length - 1 - i] = right_str[right_str.length - 1 - i];
+ }
+
+ out_str = out_str.toUpperCase();
+ return out_str;
+}
+
+module.exports = {
+ generate_identifier,
+ generate_random_code,
+};
diff --git a/packages/backend/src/util/linux.js b/packages/backend/src/util/linux.js
new file mode 100644
index 00000000..546be561
--- /dev/null
+++ b/packages/backend/src/util/linux.js
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const SmolUtil = require("./smolutil");
+
+const parse_meminfo = text => {
+ const lines = text.split('\n');
+
+ let meminfo = {};
+
+ for ( const line of lines ) {
+ if ( line.trim().length == 0 ) continue;
+
+ const [key, value_and_unit] = SmolUtil.split(line, ':', { trim: true });
+ const [value, _] = SmolUtil.split(value_and_unit, ' ', { trim: true });
+ // note: unit is always 'kB' so we discard it
+ meminfo[key] = Number.parseInt(value);
+ }
+
+ return meminfo;
+}
+
+module.exports = {
+ parse_meminfo,
+};
diff --git a/packages/backend/src/util/listenerutil.js b/packages/backend/src/util/listenerutil.js
new file mode 100644
index 00000000..711758a7
--- /dev/null
+++ b/packages/backend/src/util/listenerutil.js
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class MultiDetachable {
+ constructor() {
+ this.delegates = [];
+ this.detached_ = false;
+ }
+
+ add (delegate) {
+ if ( this.detached_ ) {
+ delegate.detach();
+ return;
+ }
+
+ this.delegates.push(delegate);
+ }
+
+ detach () {
+ this.detached_ = true;
+ for ( const delegate of this.delegates ) {
+ delegate.detach();
+ }
+ }
+}
+
+class AlsoDetachable {
+ constructor () {
+ this.also = () => {};
+ }
+
+ also (also) {
+ this.also = also;
+ return this;
+ }
+
+ detach () {
+ this.detach_();
+ this.also();
+ }
+}
+
+// TODO: this doesn't work, but I don't know why yet.
+class RemoveFromArrayDetachable extends AlsoDetachable {
+ constructor (array, element) {
+ super();
+ this.array = array;
+ this.element = element;
+ }
+
+ detach_ () {
+ for ( let i=0; i < 10; i++ ) console.log('THIS DOES GET CALLED');
+ const index = this.array.indexOf(this.element);
+ if ( index !== -1 ) {
+ this.array.splice(index, 1);
+ }
+ }
+}
+
+module.exports = {
+ MultiDetachable,
+ RemoveFromArrayDetachable,
+};
diff --git a/packages/backend/src/util/lockutil.js b/packages/backend/src/util/lockutil.js
new file mode 100644
index 00000000..b0862600
--- /dev/null
+++ b/packages/backend/src/util/lockutil.js
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { TeePromise } = require("./promise");
+
+class RWLock {
+ static TYPE_READ = Symbol('read');
+ static TYPE_WRITE = Symbol('write');
+
+ constructor () {
+ this.queue = [];
+
+ this.readers_ = 0;
+ this.writer_ = false;
+
+ this.on_empty_ = () => {};
+
+ this.mode = this.constructor.TYPE_READ;
+ }
+ get effective_mode () {
+ if ( this.readers_ > 0 ) return this.constructor.TYPE_READ;
+ if ( this.writer_ ) return this.constructor.TYPE_WRITE;
+ return undefined;
+ }
+ push_ (item) {
+ if ( this.readers_ === 0 && ! this.writer_ ) {
+ this.mode = item.type;
+ }
+ this.queue.push(item);
+ this.check_queue_();
+ }
+ check_queue_ () {
+ // console.log('check_queue_', {
+ // readers_: this.readers_,
+ // writer_: this.writer_,
+ // queue: this.queue.map(item => item.type),
+ // });
+ if ( this.queue.length === 0 ) {
+ if ( this.readers_ === 0 && ! this.writer_ ) {
+ this.on_empty_();
+ }
+ return;
+ }
+
+ const peek = () => this.queue[0];
+
+ if ( this.readers_ === 0 && ! this.writer_ ) {
+ this.mode = peek().type;
+ }
+
+ if ( this.mode === this.constructor.TYPE_READ ) {
+ while ( peek()?.type === this.constructor.TYPE_READ ) {
+ const item = this.queue.shift();
+ this.readers_++;
+ (async () => {
+ await item.p_unlock;
+ this.readers_--;
+ this.check_queue_();
+ })();
+ item.p_operation.resolve();
+ }
+ return;
+ }
+
+ if ( this.writer_ ) return;
+
+ const item = this.queue.shift();
+ this.writer_ = true;
+ (async () => {
+ await item.p_unlock;
+ this.writer_ = false;
+ this.check_queue_();
+ })();
+ item.p_operation.resolve();
+ }
+ async rlock () {
+ const p_read = new TeePromise();
+ const p_unlock = new TeePromise();
+ const handle = {
+ unlock: () => {
+ p_unlock.resolve();
+ }
+ };
+
+ this.push_({
+ type: this.constructor.TYPE_READ,
+ p_operation: p_read,
+ p_unlock,
+ });
+ await p_read;
+
+ return handle;
+ }
+
+ async wlock () {
+ const p_write = new TeePromise();
+ const p_unlock = new TeePromise();
+ const handle = {
+ unlock: () => {
+ p_unlock.resolve();
+ }
+ };
+
+ this.push_({
+ type: this.constructor.TYPE_WRITE,
+ p_operation: p_write,
+ p_unlock,
+ });
+ await p_write;
+
+ return handle;
+ }
+
+}
+
+module.exports = {
+ RWLock,
+};
diff --git a/packages/backend/src/util/multivalue.js b/packages/backend/src/util/multivalue.js
new file mode 100644
index 00000000..27d453f8
--- /dev/null
+++ b/packages/backend/src/util/multivalue.js
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { AdvancedBase } = require("puter-js-common");
+
+/**
+ * MutliValue represents a subject with multiple values or a value with multiple
+ * formats/types. It can be used for lazy evaluation of values and prioritizing
+ * equally-suitable outputs with lower resource cost.
+ *
+ * For example, a MultiValue representing a file could have a key called
+ * `stream` as well as a key called `s3-info`. It would always be possible
+ * to obtain a `stream` but when the `s3-info` is available and applicable
+ * it will be less costly to obtain.
+ */
+class MultiValue extends AdvancedBase {
+ constructor () {
+ super();
+ this.factories = {};
+ this.values = {};
+ }
+
+ async add_factory (key_desired, key_available, fn, cost) {
+ if ( ! this.factories[key_desired] ) {
+ this.factories[key_desired] = [];
+ }
+ this.factories[key_desired].push({
+ key_available,
+ fn,
+ cost,
+ });
+ }
+
+ async get (key) {
+ return this._get(key);
+ }
+
+ set (key, value) {
+ this.values[key] = value;
+ }
+
+ async _get (key) {
+ if ( this.values[key] ) {
+ return this.values[key];
+ }
+ const factories = this.factories[key];
+ if ( ! factories || ! factories.length ) {
+ console.log('no factory for key', key)
+ return undefined;
+ }
+ for ( const factory of factories ) {
+ const available = await this._get(factory.key_available);
+ if ( ! available ) {
+ console.log('no available for key', key, factory.key_available);
+ continue;
+ }
+ const value = await factory.fn(available);
+ this.values[key] = value;
+ return value;
+ }
+ return undefined;
+ }
+}
+
+module.exports = {
+ MultiValue,
+};
diff --git a/packages/backend/src/util/opmath.js b/packages/backend/src/util/opmath.js
new file mode 100644
index 00000000..a9094736
--- /dev/null
+++ b/packages/backend/src/util/opmath.js
@@ -0,0 +1,201 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class Getter {
+ static adapt (v) {
+ if ( typeof v === 'function' ) return v;
+ return () => v;
+ }
+}
+
+const LinearByCountGetter = ({ initial, slope, pre = false }) => {
+ let value = initial;
+ return () => {
+ if ( pre ) value += slope;
+ let v = value;
+ if ( ! pre ) value += slope;
+ return v;
+ }
+}
+
+const ConstantGetter = ({ initial }) => () => initial;
+
+// bind function for parameterized functions
+const Bind = (fn, important_parameters) => {
+ return (given_parameters) => {
+ return fn({
+ ...given_parameters,
+ ...important_parameters,
+ });
+ }
+}
+
+/**
+ * SwitchByCountGetter
+ *
+ * @example
+ * const getter = SwitchByCountGetter({
+ * initial: 0,
+ * body: {
+ * 0: Bind(LinearByCountGetter, { slop: 1 }),
+ * 5: ConstantGetter,
+ * }
+ * }); // 0, 1, 2, 3, 4, 4, 4, ...
+ */
+const SwitchByCountGetter = ({ initial, body }) => {
+ let value = initial ?? 0;
+ let count = 0;
+ let getter;
+ if ( ! body.hasOwnProperty(count) ) {
+ throw new Error('body of SwitchByCountGetter must have an entry for count 0');
+ }
+ return () => {
+ if ( body.hasOwnProperty(count) ) {
+ getter = body[count]({ initial: value });
+ console.log('getter is', getter)
+ }
+ value = getter();
+ count++;
+ return value;
+ }
+}
+
+class StreamReducer {
+ constructor (initial) {
+ this.value = initial;
+ }
+
+ put (v) {
+ this._put(v);
+ }
+
+ get () {
+ return this._get();
+ }
+
+ _put (v) {
+ throw new Error('Not implemented');
+ }
+
+ _get () {
+ return this.value;
+ }
+}
+
+class EWMA extends StreamReducer {
+ constructor ({ initial, alpha }) {
+ super(initial ?? 0);
+ console.log('VALL', this.value)
+ this.alpha = Getter.adapt(alpha);
+ }
+
+ _put (v) {
+ this.value = this.alpha() * v + (1 - this.alpha()) * this.value;
+ }
+}
+
+class MovingMode extends StreamReducer {
+ constructor ({ initial, window_size }) {
+ super(initial ?? 0);
+ this.window_size = window_size ?? 30;
+ this.window = [];
+ }
+
+ _put (v) {
+ this.window.push(v);
+ if ( this.window.length > this.window_size ) {
+ this.window.shift();
+ }
+ this.value = this._get_mode();
+ }
+
+ _get_mode () {
+ let counts = {};
+ for ( let v of this.window ) {
+ if ( ! counts.hasOwnProperty(v) ) counts[v] = 0;
+ counts[v]++;
+ }
+ let max = 0;
+ let mode = null;
+ for ( let v in counts ) {
+ if ( counts[v] > max ) {
+ max = counts[v];
+ mode = v;
+ }
+ }
+ return mode;
+ }
+}
+
+class TimeWindow {
+ constructor ({ window_duration, reducer }) {
+ this.window_duration = window_duration;
+ this.reducer = reducer;
+ this.entries_ = [];
+ }
+
+ add (value) {
+ this.remove_stale_entries_();
+
+ const timestamp = Date.now();
+ this.entries_.push({
+ timestamp,
+ value,
+ });
+ }
+
+ get () {
+ this.remove_stale_entries_();
+
+ const values = this.entries_.map(entry => entry.value);
+ if ( ! this.reducer ) return values;
+
+ return this.reducer(values);
+ }
+
+ get_entries () {
+ return [...this.entries_];
+ }
+
+ remove_stale_entries_ () {
+ let i = 0;
+ const current_ts = Date.now();
+ for ( ; i < this.entries_.length ; i++ ) {
+ const entry = this.entries_[i];
+ // as soon as an entry is in the window we can break,
+ // since entries will always be in ascending order by timestamp
+ if ( current_ts - entry.timestamp < this.window_duration ) {
+ break;
+ }
+ }
+
+ this.entries_ = this.entries_.slice(i);
+ }
+}
+
+module.exports = {
+ Getter,
+ LinearByCountGetter,
+ SwitchByCountGetter,
+ ConstantGetter,
+ Bind,
+ StreamReducer,
+ EWMA,
+ MovingMode,
+ TimeWindow,
+}
diff --git a/packages/backend/src/util/otelutil.js b/packages/backend/src/util/otelutil.js
new file mode 100644
index 00000000..ecebdb46
--- /dev/null
+++ b/packages/backend/src/util/otelutil.js
@@ -0,0 +1,158 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+// The OpenTelemetry SDK provides a very error-prone API for creating
+// spans. This is a wrapper around the SDK that makes it convenient
+// to create spans correctly. The path of least resistance should
+// be the correct path, not a way to shoot yourself in the foot.
+
+const { context, trace, SpanStatusCode } = require('@opentelemetry/api');
+const { Context } = require('./context');
+const { TeePromise } = require('./promise');
+
+/*
+parallel span example from GPT-4:
+
+promises.push(tracer.startActiveSpan(`job:${job.id}`, (span) => {
+ return context.with(trace.setSpan(context.active(), span), async () => {
+ try {
+ await job.run();
+ } catch (error) {
+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+ throw error;
+ } finally {
+ span.end();
+ }
+ });
+}));
+*/
+
+const spanify = (label, fn) => async (...args) => {
+ const context = Context.get();
+ if ( ! context ) {
+ // We don't use the proper logger here because we would normally
+ // be getting it from context
+ console.error('spanify failed', new Error('missing context'));
+ }
+
+ const tracer = context.get('services').get('traceService').tracer;
+ let result;
+ await tracer.startActiveSpan(label, async span => {
+ result = await fn(...args);
+ span.end();
+ });
+ return result;
+};
+
+const abtest = async (label, impls) => {
+ const context = Context.get();
+ if ( ! context ) {
+ // We don't use the proper logger here because we would normally
+ // be getting it from context
+ console.error('abtest failed', new Error('missing context'));
+ }
+
+ const tracer = context.get('services').get('traceService').tracer;
+ let result;
+ const impl_keys = Object.keys(impls);
+ const impl_i = Math.floor(Math.random() * impl_keys.length);
+ const impl_name = impl_keys[impl_i];
+ const impl = impls[impl_name]
+
+ await tracer.startActiveSpan(label + ':' + impl_name, async span => {
+ span.setAttribute('abtest.impl', impl_name);
+ result = await impl();
+ span.end();
+ });
+ return result;
+};
+
+class ParallelTasks {
+ constructor ({ tracer, max } = {}) {
+ this.tracer = tracer;
+ this.max = max ?? Infinity;
+ this.promises = [];
+
+ this.queue_ = [];
+ this.ongoing_ = 0;
+ }
+
+ add (name, fn, flags) {
+ if ( this.ongoing_ >= this.max && ! flags?.force ) {
+ const p = new TeePromise();
+ this.promises.push(p);
+ this.queue_.push([name, fn, p]);
+ return;
+ }
+
+ // const span = this.tracer.startSpan(name);
+ this.promises.push(this.run_(name, fn));
+ }
+
+ run_ (name, fn) {
+ this.ongoing_++;
+ const span = this.tracer.startSpan(name);
+ return context.with(trace.setSpan(context.active(), span), async () => {
+ try {
+ console.log('AA :: BEFORE');
+ const res = await fn();
+ console.log('AA :: AFTER');
+ this.ongoing_--;
+ this.check_queue_();
+ return res;
+ } catch (error) {
+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+ throw error;
+ } finally {
+ span.end();
+ }
+ })
+ }
+
+ check_queue_ () {
+ console.log('CHECKING QUQUE', this.ongoing_, this.queue_.length);
+ while ( this.ongoing_ < this.max && this.queue_.length > 0 ) {
+ const [name, fn, p] = this.queue_.shift();
+ const run_p = this.run_(name, fn);
+ run_p.then(p.resolve.bind(p), p.reject.bind(p));
+ }
+ }
+
+ async awaitAll () {
+ await Promise.all(this.promises);
+ }
+
+ async awaitAllAndDeferThrow () {
+ const results = await Promise.allSettled(this.promises);
+ const errors = [];
+ for ( const result of results ) {
+ if ( result.status === 'rejected' ) {
+ errors.push(result.reason);
+ }
+ }
+ if ( errors.length !== 0 ) {
+ throw new AggregateError(errors);
+ }
+ }
+}
+
+module.exports = {
+ ParallelTasks,
+ spanify,
+ abtest,
+};
diff --git a/packages/backend/src/util/promise.js b/packages/backend/src/util/promise.js
new file mode 100644
index 00000000..137b19c3
--- /dev/null
+++ b/packages/backend/src/util/promise.js
@@ -0,0 +1,156 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class TeePromise {
+ static STATUS_PENDING = Symbol('pending');
+ static STATUS_RUNNING = {};
+ static STATUS_DONE = Symbol('done');
+ constructor () {
+ this.status_ = this.constructor.STATUS_PENDING;
+ this.donePromise = new Promise((resolve, reject) => {
+ this.doneResolve = resolve;
+ this.doneReject = reject;
+ });
+ }
+ get status () {
+ return this.status_;
+ }
+ set status (status) {
+ this.status_ = status;
+ if ( status === this.constructor.STATUS_DONE ) {
+ this.doneResolve();
+ }
+ }
+ resolve (value) {
+ this.status_ = this.constructor.STATUS_DONE;
+ this.doneResolve(value);
+ }
+ awaitDone () {
+ return this.donePromise;
+ }
+ then (fn, ...a) {
+ return this.donePromise.then(fn, ...a);
+ }
+
+ reject (err) {
+ this.status_ = this.constructor.STATUS_DONE;
+ this.doneReject(err);
+ }
+
+ /**
+ * @deprecated use then() instead
+ */
+ onComplete(fn) {
+ return this.then(fn);
+ }
+}
+
+class Lock {
+ constructor() {
+ this._locked = false;
+ this._waiting = [];
+ }
+
+ async acquire(callback) {
+ await new Promise(resolve => {
+ if ( ! this._locked ) {
+ this._locked = true;
+ resolve();
+ } else {
+ this._waiting.push({
+ resolve,
+ });
+ }
+ })
+ if ( callback ) {
+ let retval;
+ try {
+ retval = await callback();
+ } finally {
+ this.release();
+ }
+ return retval;
+ }
+ }
+
+ release() {
+ if (this._waiting.length > 0) {
+ const { resolve } = this._waiting.shift();
+ resolve();
+ } else {
+ this._locked = false;
+ }
+ }
+}
+
+/**
+ * @callback behindScheduleCallback
+ * @param {number} drift - The number of milliseconds that the callback was
+ * called behind schedule.
+ * @returns {boolean} - If the callback returns true, the timer will be
+ * cancelled.
+ */
+
+/**
+ * When passing an async callback to setInterval, it's possible for the
+ * callback to be called again before the previous invocation has finished.
+ *
+ * This function wraps setInterval and ensures that the callback is not
+ * called again until the previous invocation has finished.
+ *
+ * @param {Function} callback - The function to call when the timer elapses.
+ * @param {number} delay - The minimum number of milliseconds between invocations.
+ * @param {?Array} args - Additional arguments to pass to setInterval.
+ * @param {?Object} options - Additional options.
+ * @param {behindScheduleCallback} options.onBehindSchedule - A callback to call when the callback is called behind schedule.
+ */
+const asyncSafeSetInterval = async (callback, delay, args, options) => {
+ args = args ?? [];
+ options = options ?? {};
+ const { onBehindSchedule } = options;
+
+ const sleep = (ms) => new Promise(rslv => setTimeout(rslv, ms));
+
+ for ( ;; ) {
+ await sleep(delay);
+
+ const ts_start = Date.now();
+ await callback(...args);
+ const ts_end = Date.now();
+
+ const runtime = ts_end - ts_start;
+ const sleep_time = delay - runtime;
+
+ if ( sleep_time < 0 ) {
+ if ( onBehindSchedule ) {
+ const cancel = await onBehindSchedule(-sleep_time);
+ if ( cancel ) {
+ return;
+ }
+ }
+ } else {
+ await sleep(sleep_time);
+ }
+ }
+}
+
+module.exports = {
+ TeePromise,
+ Lock,
+ asyncSafeSetInterval,
+};
diff --git a/packages/backend/src/util/queuing.js b/packages/backend/src/util/queuing.js
new file mode 100644
index 00000000..47955c28
--- /dev/null
+++ b/packages/backend/src/util/queuing.js
@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class QueueBatcher {
+ //
+}
diff --git a/packages/backend/src/util/retryutil.js b/packages/backend/src/util/retryutil.js
new file mode 100644
index 00000000..0bdc614f
--- /dev/null
+++ b/packages/backend/src/util/retryutil.js
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+/**
+ * Retries a function a maximum number of times, with a given interval between each try.
+ * @param {Function} func - The function to retry
+ * @param {Number} max_tries - The maximum number of tries
+ * @param {Number} interval - The interval between each try
+ * @returns {Promise<[Error, Boolean, any]>} - A promise that resolves to an
+ * array containing the last error, a boolean indicating whether the function
+ * eventually succeeded, and the return value of the function
+ */
+const simple_retry = async function simple_retry (func, max_tries, interval) {
+ let tries = 0;
+ let last_error = null;
+
+ if ( max_tries === undefined ) {
+ throw new Error('simple_retry: max_tries is undefined');
+ }
+ if ( interval === undefined ) {
+ throw new Error('simple_retry: interval is undefined');
+ }
+
+ while ( tries < max_tries ) {
+ try {
+ return [last_error, true, await func()];
+ } catch ( error ) {
+ last_error = error;
+ tries++;
+ await new Promise((resolve) => setTimeout(resolve, interval));
+ }
+ }
+ if ( last_error === null ) {
+ last_error = new Error('simple_retry: failed, but error is null');
+ }
+ return [last_error, false];
+};
+
+const poll = async function poll({ poll_fn, schedule_fn }) {
+ let delay = undefined;
+
+ while ( true ) {
+ const is_done = await poll_fn();
+ if ( is_done ) {
+ return;
+ }
+ delay = schedule_fn(delay);
+ await new Promise((resolve) => setTimeout(resolve, delay));
+ }
+}
+
+module.exports = {
+ simple_retry,
+ poll,
+};
diff --git a/packages/backend/src/util/smolutil.js b/packages/backend/src/util/smolutil.js
new file mode 100644
index 00000000..12942eaf
--- /dev/null
+++ b/packages/backend/src/util/smolutil.js
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+// All of these utilities are trivial and just make the code look nicer.
+class SmolUtil {
+ // Array coercion
+ static ensure_array (value) {
+ return Array.isArray(value) ? value : [value];
+ }
+ // Variadic sum
+ static add (...a) {
+ return a.reduce((a, b) => a + b, 0);
+ }
+ static split (str, sep, options = {}) {
+ options = options || {};
+ const { trim, discard_empty } = options;
+
+ const operations = [];
+
+ if ( options.trim ) {
+ operations.push(a => a.map(str => str.trim()));
+ }
+
+ if ( options.discard_empty ) {
+ operations.push(a => a.filter(str => str.length > 0));
+ }
+
+ let result = str.split(sep);
+ for ( const operation of operations ) {
+ result = operation(result);
+ }
+ return result;
+ }
+}
+
+module.exports = SmolUtil;
diff --git a/packages/backend/src/util/stdioutil.js b/packages/backend/src/util/stdioutil.js
new file mode 100644
index 00000000..1cfd99db
--- /dev/null
+++ b/packages/backend/src/util/stdioutil.js
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+/**
+ * Strip ANSI escape sequences from a string (e.g. color codes)
+ * and then return the length of the resulting string.
+ *
+ * @param {*} str
+ */
+const visible_length = (str) => {
+ return str.replace(/\x1b\[[0-9;]*m/g, '').length;
+};
+
+/**
+ * Split a string into lines according to the terminal width,
+ * preserving ANSI escape sequences, and return an array of lines.
+ *
+ * @param {*} str
+ */
+const split_lines = (str) => {
+ const lines = [];
+ let line = '';
+ let line_length = 0;
+ for (const c of str) {
+ line += c;
+ if (c === '\n') {
+ lines.push(line);
+ line = '';
+ line_length = 0;
+ } else {
+ line_length++;
+ if (line_length >= process.stdout.columns) {
+ lines.push(line);
+ line = '';
+ line_length = 0;
+ }
+ }
+ }
+ if (line.length) {
+ lines.push(line);
+ }
+ return lines;
+};
+
+
+module.exports = {
+ visible_length,
+ split_lines,
+};
diff --git a/packages/backend/src/util/streamutil.js b/packages/backend/src/util/streamutil.js
new file mode 100644
index 00000000..d6b2596a
--- /dev/null
+++ b/packages/backend/src/util/streamutil.js
@@ -0,0 +1,467 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const { PassThrough, Readable, Transform } = require('stream');
+const { TeePromise } = require('./promise');
+const { EWMA } = require('./opmath');
+
+class StreamBuffer extends TeePromise {
+ constructor () {
+ super();
+
+ this.stream = new PassThrough();
+ this.buffer_ = '';
+
+ this.stream.on('data', (chunk) => {
+ this.buffer_ += chunk.toString();
+ });
+
+ this.stream.on('end', () => {
+ this.resolve(this.buffer_);
+ });
+
+ this.stream.on('error', (err) => {
+ this.reject(err);
+ });
+ }
+}
+
+const stream_to_the_void = stream => {
+ stream.on('data', () => {});
+ stream.on('end', () => {});
+ stream.on('error', () => {});
+};
+
+const pausing_tee = (source, n) => {
+ const { PassThrough } = require('stream');
+
+ const ready_ = [];
+ const streams_ = [];
+ let first_ = true;
+ for ( let i=0 ; i < n ; i++ ) {
+ ready_.push(true);
+ const stream = new PassThrough();
+ streams_.push(stream);
+ stream.on('drain', () => {
+ ready_[i] = true;
+ // console.log(source.id, 'PR :: drain from reader', i, ready_);
+ if ( first_ ) {
+ source.resume();
+ first_ = false;
+ }
+ if (ready_.every(v => !! v)) source.resume();
+ });
+ // stream.on('newListener', (event, listener) => {
+ // console.log('PR :: newListener', i, event, listener);
+ // });
+ }
+
+ source.on('data', (chunk) => {
+ // console.log(source.id, 'PT :: data from source', chunk.length);
+ ready_.forEach((v, i) => {
+ ready_[i] = streams_[i].write(chunk);
+ });
+ if ( ! ready_.every(v => !! v) ) {
+ // console.log('PT :: pausing source', ready_);
+ source.pause();
+ return;
+ }
+ });
+
+ source.on('end', () => {
+ // console.log(source.id, 'PT :: end from source');
+ for ( let i=0 ; i < n ; i++ ) {
+ streams_[i].end();
+ }
+ });
+
+ source.on('error', (err) => {
+ // console.log(source.id, 'PT :: error from source', err);
+ for ( let i=0 ; i < n ; i++ ) {
+ streams_[i].emit('error', err);
+ }
+ });
+
+ return streams_;
+};
+
+class LoggingStream extends Transform {
+ constructor(options) {
+ super(options);
+ this.count = 0;
+ }
+
+ _transform(chunk, encoding, callback) {
+ const stream_id = this.id ?? 'unknown';
+ console.log(`[DATA@${stream_id}] :: ${chunk.length} (${this.count++})`);
+ this.push(chunk);
+ callback();
+ }
+}
+
+// logs stream activity
+const logging_stream = source => {
+ const stream = new LoggingStream();
+ if ( source.id ) stream.id = source.id;
+ source.pipe(stream);
+ return stream;
+};
+
+/**
+ * Returns a readable stream that emits the data from `originalDataStream`,
+ * replacing the data at position `offset` with the data from `newDataStream`.
+ * When the `newDataStream` is consumed, the `originalDataStream` will continue
+ * emitting data.
+ *
+ * Note: `originalDataStream` will be paused until `newDataStream` is consumed.
+ *
+ * @param {*} originalDataStream
+ * @param {*} newDataStream
+ * @param {*} offset
+ */
+const offset_write_stream = ({
+ originalDataStream, newDataStream, offset,
+ replace_length = 0,
+}) => {
+ const passThrough = new PassThrough();
+ let remaining = offset;
+ let new_end = false;
+ let org_end = false;
+ let replaced_bytes = 0;
+
+ let last_state = null;
+ const implied = {
+ get state () {
+ const state =
+ remaining > 0 ? STATE_ORIGINAL_STREAM :
+ new_end && org_end ? STATE_END :
+ new_end ? STATE_CONTINUE :
+ STATE_NEW_STREAM ;
+ // (comment to reset indentation)
+ if ( state !== last_state ) {
+ last_state = state;
+ if ( state.on_enter ) state.on_enter();
+ }
+ return state;
+ }
+ };
+
+ let defer_buffer = Buffer.alloc(0);
+ let new_stream_early_buffer = Buffer.alloc(0);
+
+ const original_stream_on_data = chunk => {
+ console.log('original stream data', chunk.length, implied.state);
+ console.log('received from original:', chunk.toString());
+
+ if ( implied.state === STATE_NEW_STREAM ) {
+ console.warn('original stream is not paused');
+ defer_buffer = Buffer.concat([defer_buffer, chunk]);
+ return;
+ }
+
+ if (
+ implied.state === STATE_ORIGINAL_STREAM &&
+ chunk.length >= remaining
+ ) {
+ defer_buffer = chunk.slice(remaining);
+ console.log('deferred:', defer_buffer.toString());
+ chunk = chunk.slice(0, remaining);
+ }
+
+ if (
+ implied.state === STATE_CONTINUE &&
+ replaced_bytes < replace_length
+ ) {
+ const remaining_replacement = replace_length - replaced_bytes;
+ if ( chunk.length <= remaining_replacement ) {
+ console.log('skipping chunk', chunk.toString());
+ replaced_bytes += chunk.length;
+ return; // skip the chunk
+ }
+ console.log('skipping part of chunk', chunk.slice(0, remaining_replacement).toString());
+ chunk = chunk.slice(remaining_replacement);
+
+ // `+= remaining_replacement` and `= replace_length` are equivalent
+ // at this point.
+ replaced_bytes += remaining_replacement;
+ }
+
+ remaining -= chunk.length;
+ console.log('pushing from org stream:', chunk.toString());
+ passThrough.push(chunk);
+ implied.state;
+ }
+
+ const STATE_ORIGINAL_STREAM = {
+ on_enter: () => {
+ console.log('STATE_ORIGINAL_STREAM');
+ newDataStream.pause();
+ }
+ };
+ const STATE_NEW_STREAM = {
+ on_enter: () => {
+ console.log('STATE_NEW_STREAM');
+ originalDataStream.pause();
+ originalDataStream.off('data', original_stream_on_data);
+ newDataStream.resume();
+ },
+ };
+ const STATE_CONTINUE = {
+ on_enter: () => {
+ console.log('STATE_CONTINUE');
+ if ( defer_buffer.length > 0 ) {
+ const remaining_replacement = replace_length - replaced_bytes;
+ if ( replaced_bytes < replace_length ) {
+ if ( defer_buffer.length <= remaining_replacement ) {
+ console.log('skipping deferred', defer_buffer.toString());
+ replaced_bytes += defer_buffer.length;
+ defer_buffer = Buffer.alloc(0);
+ } else {
+ console.log('skipping deferred', defer_buffer.slice(0, remaining_replacement).toString());
+ defer_buffer = defer_buffer.slice(remaining_replacement);
+ replaced_bytes += remaining_replacement;
+ }
+ }
+ console.log('pushing deferred:', defer_buffer.toString());
+ passThrough.push(defer_buffer);
+ }
+ // originalDataStream.pipe(passThrough);
+ originalDataStream.on('data', original_stream_on_data);
+ originalDataStream.resume();
+ },
+ };
+ const STATE_END = {
+ on_enter: () => {
+ console.log('STATE_END');
+ passThrough.end();
+ }
+ };
+
+ implied.state;
+
+ originalDataStream.on('data', original_stream_on_data);
+ originalDataStream.on('end', () => {
+ console.log('original stream end');
+ org_end = true;
+ implied.state;
+ })
+
+ newDataStream.on('data', chunk => {
+ console.log('new stream data', chunk.toString());
+
+ if ( implied.state === STATE_NEW_STREAM ) {
+ console.log('pushing from new stream', chunk.toString());
+ passThrough.push(chunk);
+ return;
+ }
+
+ console.warn('new stream is not paused');
+ new_stream_early_buffer = Buffer.concat([new_stream_early_buffer, chunk]);
+ });
+ newDataStream.on('end', () => {
+ console.log('new stream end', implied.state);
+
+ new_end = true;
+ implied.state;
+ });
+
+ return passThrough;
+};
+
+
+class ProgressReportingStream extends Transform {
+ constructor(options, { total, progress_callback }) {
+ super(options);
+ this.total = total;
+ this.loaded = 0;
+ this.progress_callback = progress_callback;
+ }
+
+ _transform(chunk, encoding, callback) {
+ this.loaded += chunk.length;
+ this.progress_callback({
+ loaded: this.loaded,
+ uploaded: this.loaded,
+ total: this.total,
+ });
+ this.push(chunk);
+ callback();
+ }
+}
+
+const progress_stream = (source, { total, progress_callback }) => {
+ const stream = new ProgressReportingStream({}, { total, progress_callback });
+ source.pipe(stream);
+ return stream;
+}
+
+class StuckDetectorStream extends Transform {
+ constructor(options, {
+ timeout,
+ on_stuck,
+ on_unstuck,
+ }) {
+ super(options);
+ this.timeout = timeout;
+ this.stuck_ = false;
+ this.on_stuck = on_stuck;
+ this.on_unstuck = on_unstuck;
+ this.last_chunk_time = Date.now();
+
+ this._start_timer();
+ }
+
+ _start_timer () {
+ if ( this.timer ) clearTimeout(this.timer);
+ this.timer = setTimeout(() => {
+ if ( this.stuck_ ) return;
+ this.stuck_ = true;
+ this.on_stuck();
+ }, this.timeout);
+ }
+
+ _transform(chunk, encoding, callback) {
+ if ( this.stuck_ ) {
+ this.stuck_ = false;
+ this.on_unstuck();
+ }
+ this._start_timer();
+ this.push(chunk);
+ callback();
+ }
+
+ _flush(callback) {
+ clearTimeout(this.timer);
+ callback();
+ }
+}
+
+const stuck_detector_stream = (source, {
+ timeout,
+ on_stuck,
+ on_unstuck,
+}) => {
+ const stream = new StuckDetectorStream({}, {
+ timeout,
+ on_stuck,
+ on_unstuck,
+ });
+ source.pipe(stream);
+ return stream;
+}
+
+string_to_stream = (str, chunk_size) => {
+ const s = new Readable();
+ s._read = () => {}; // redundant? see update below
+ // split string into chunks
+ const chunks = [];
+ for (let i = 0; i < str.length; i += chunk_size) {
+ chunks.push(str.slice(i, Math.min(i + chunk_size, str.length)));
+ }
+ // push each chunk onto the readable stream
+ chunks.forEach((chunk) => {
+ s.push(chunk);
+ });
+ s.push(null);
+ return s;
+};
+
+async function* chunk_stream(
+ stream,
+ chunk_size = 1024 * 1024 * 5,
+ expected_chunk_time,
+) {
+ let buffer = Buffer.alloc(chunk_size);
+ let offset = 0;
+
+ const chunk_time_ewma = expected_chunk_time !== undefined
+ ? expected_chunk_time
+ : null;
+
+ for await (const chunk of stream) {
+ if ( globalThis.average_chunk_size ) {
+ globalThis.average_chunk_size.put(chunk.length);
+ }
+ let remaining = chunk_size - offset;
+ let amount = Math.min(remaining, chunk.length);
+
+ chunk.copy(buffer, offset, 0, amount);
+ offset += amount;
+
+ while (offset >= chunk_size) {
+ console.log('start yield');
+ yield buffer;
+ console.log('end yield');
+
+ buffer = Buffer.alloc(chunk_size);
+ offset = 0;
+
+ if (amount < chunk.length) {
+ const leftover = chunk.length - amount;
+ const next_amount = Math.min(leftover, chunk_size);
+ chunk.copy(buffer, offset, amount, amount + next_amount);
+ offset += next_amount;
+ amount += next_amount;
+ }
+ }
+
+ if ( chunk_time_ewma !== null ) {
+ const chunk_time = chunk_time_ewma.get();
+ // const sleep_time = chunk_size * chunk_time;
+ const sleep_time = (chunk.length / chunk_size) * chunk_time / 2;
+ // const sleep_time = (amount / chunk_size) * chunk_time;
+ // const sleep_time = (amount / chunk_size) * chunk_time;
+ console.log(`start sleep ${amount} / ${chunk_size} * ${chunk_time} = ${sleep_time}`);
+ await new Promise(resolve => setTimeout(resolve, sleep_time));
+ console.log('end sleep');
+ }
+ }
+
+ if (offset > 0) {
+ yield buffer.subarray(0, offset); // Yield remaining chunk if it's not empty.
+ }
+}
+
+const stream_to_buffer = async (stream) => {
+ const chunks = [];
+ for await (const chunk of stream) {
+ chunks.push(chunk);
+ }
+ return Buffer.concat(chunks);
+};
+
+const buffer_to_stream = (buffer) => {
+ const stream = new Readable();
+ stream.push(buffer);
+ stream.push(null);
+ return stream;
+};
+
+module.exports = {
+ StreamBuffer,
+ stream_to_the_void,
+ pausing_tee,
+ logging_stream,
+ offset_write_stream,
+ progress_stream,
+ stuck_detector_stream,
+ string_to_stream,
+ chunk_stream,
+ stream_to_buffer,
+ buffer_to_stream,
+};
diff --git a/packages/backend/src/util/strutil.js b/packages/backend/src/util/strutil.js
new file mode 100644
index 00000000..c520836e
--- /dev/null
+++ b/packages/backend/src/util/strutil.js
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+// Convenience function for quoting strings in error messages.
+// Turns a string like this: some`value`
+// Into a string like this: `some\`value\``
+const quot = (str) => {
+ if ( str === undefined ) return '[undefined]';
+ if ( str === null ) return '[null]';
+ if ( typeof str === 'function' ) return '[function]';
+ if ( typeof str === 'object' ) return '[object]';
+ if ( typeof str === 'number' ) return '(' + str + ')';
+
+ str = '' + str;
+
+ str = str.replace(/["`]/g, m => m === '"' ? "`" : '"');
+ str = JSON.stringify('' + str);
+ str = str.replace(/["`]/g, m => m === '"' ? "`" : '"');
+ return str;
+}
+
+const osclink = (url, text) => {
+ if ( ! text ) text = url;
+ return `\x1B]8;;${url}\x1B\\${text}\x1B]8;;\x1B\\`;
+}
+
+format_as_usd = (amount) => {
+ if ( amount < 0.01 ) {
+ if ( amount < 0.00001 ) {
+ // scientific notation
+ return '$' + amount.toExponential(2);
+ }
+ return '$' + amount.toFixed(5);
+ }
+ return '$' + amount.toFixed(2);
+}
+
+module.exports = {
+ quot,
+ osclink,
+ format_as_usd,
+};
\ No newline at end of file
diff --git a/packages/backend/src/util/time.js b/packages/backend/src/util/time.js
new file mode 100644
index 00000000..17f67014
--- /dev/null
+++ b/packages/backend/src/util/time.js
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+class TimeUnit {
+ static valueOf () {
+ return this.value;
+ }
+}
+
+class MILLISECOND extends TimeUnit {
+ static value = 1;
+}
+
+class SECOND extends TimeUnit {
+ static value = 1000 * MILLISECOND;
+}
+
+class MINUTE extends TimeUnit {
+ static value = 60 * SECOND;
+}
+
+class HOUR extends TimeUnit {
+ static value = 60 * MINUTE;
+}
+
+class DAY extends TimeUnit {
+ static value = 24 * HOUR;
+}
+
+module.exports = {
+ MILLISECOND,
+ SECOND,
+ MINUTE,
+ HOUR,
+ DAY,
+};
diff --git a/packages/backend/src/util/uploadutil.js b/packages/backend/src/util/uploadutil.js
new file mode 100644
index 00000000..f3cb5950
--- /dev/null
+++ b/packages/backend/src/util/uploadutil.js
@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
diff --git a/packages/backend/src/util/urlutil.js b/packages/backend/src/util/urlutil.js
new file mode 100644
index 00000000..ad02b464
--- /dev/null
+++ b/packages/backend/src/util/urlutil.js
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2024 Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+const origin_from_url = url => {
+ try {
+ const parsedUrl = new URL(url);
+ // Origin is protocol + hostname + port
+ return `${parsedUrl.protocol}//${parsedUrl.hostname}${parsedUrl.port ? `:${parsedUrl.port}` : ''}`;
+ } catch (error) {
+ console.error('Invalid URL:', error.message);
+ return null;
+ }
+};
+
+module.exports = {
+ origin_from_url
+};
diff --git a/packages/puter-js-common/index.js b/packages/puter-js-common/index.js
new file mode 100644
index 00000000..d71fb116
--- /dev/null
+++ b/packages/puter-js-common/index.js
@@ -0,0 +1,5 @@
+const { AdvancedBase } = require('./src/AdvancedBase');
+
+module.exports = {
+ AdvancedBase,
+};
diff --git a/packages/puter-js-common/package.json b/packages/puter-js-common/package.json
new file mode 100644
index 00000000..efb1b90f
--- /dev/null
+++ b/packages/puter-js-common/package.json
@@ -0,0 +1,11 @@
+{
+ "name": "puter-js-common",
+ "version": "1.0.0",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "Puter Technologies Inc.",
+ "license": "UNLICENSED"
+}
diff --git a/packages/puter-js-common/src/AdvancedBase.js b/packages/puter-js-common/src/AdvancedBase.js
new file mode 100644
index 00000000..e0ca8ab4
--- /dev/null
+++ b/packages/puter-js-common/src/AdvancedBase.js
@@ -0,0 +1,15 @@
+// This doesn't go in ./bases because it logically depends on
+// both ./bases and ./traits, and ./traits depends on ./bases.
+
+const { TraitBase } = require("./bases/TraitBase");
+
+class AdvancedBase extends TraitBase {
+ static TRAITS = [
+ require('./traits/NodeModuleDITrait'),
+ require('./traits/PropertiesTrait'),
+ ]
+}
+
+module.exports = {
+ AdvancedBase,
+};
diff --git a/packages/puter-js-common/src/bases/BasicBase.js b/packages/puter-js-common/src/bases/BasicBase.js
new file mode 100644
index 00000000..a01a0f00
--- /dev/null
+++ b/packages/puter-js-common/src/bases/BasicBase.js
@@ -0,0 +1,37 @@
+class BasicBase {
+ _get_inheritance_chain () {
+ const chain = [];
+ let cls = this.constructor;
+ while ( cls && cls !== BasicBase ) {
+ chain.push(cls);
+ cls = cls.__proto__;
+ }
+ return chain.reverse();
+ }
+
+ _get_merged_static_array (key) {
+ const chain = this._get_inheritance_chain();
+ const values = [];
+ for ( const cls of chain ) {
+ if ( cls[key] ) {
+ values.push(...cls[key]);
+ }
+ }
+ return values;
+ }
+
+ _get_merged_static_object (key) {
+ const chain = this._get_inheritance_chain();
+ const values = {};
+ for ( const cls of chain ) {
+ if ( cls[key] ) {
+ Object.assign(values, cls[key]);
+ }
+ }
+ return values;
+ }
+}
+
+module.exports = {
+ BasicBase,
+};
\ No newline at end of file
diff --git a/packages/puter-js-common/src/bases/TraitBase.js b/packages/puter-js-common/src/bases/TraitBase.js
new file mode 100644
index 00000000..3cec906a
--- /dev/null
+++ b/packages/puter-js-common/src/bases/TraitBase.js
@@ -0,0 +1,23 @@
+const { BasicBase } = require("./BasicBase");
+
+class TraitBase extends BasicBase {
+ constructor (parameters, ...a) {
+ super(parameters, ...a);
+ for ( const trait of this.traits ) {
+ trait.install_in_instance(
+ this,
+ {
+ parameters: parameters || {},
+ }
+ )
+ }
+ }
+
+ get traits () {
+ return this._get_merged_static_array('TRAITS');
+ }
+}
+
+module.exports = {
+ TraitBase,
+};
diff --git a/packages/puter-js-common/src/traits/NodeModuleDITrait.js b/packages/puter-js-common/src/traits/NodeModuleDITrait.js
new file mode 100644
index 00000000..1e011de6
--- /dev/null
+++ b/packages/puter-js-common/src/traits/NodeModuleDITrait.js
@@ -0,0 +1,41 @@
+/**
+ * This trait allows dependency injection of node modules.
+ * This is incredibly useful for passing mock implementations
+ * of modules for unit testing.
+ *
+ * @example
+ * class MyClass extends AdvancedBase {
+ * static MODULES = {
+ * axios,
+ * };
+ * }
+ *
+ * const my_class = new MyClass({
+ * modules: {
+ * axios: MY_AXIOS_MOCK,
+ * }
+ * });
+ */
+module.exports = {
+ install_in_instance: (instance, { parameters }) => {
+ const modules = instance._get_merged_static_object('MODULES');
+
+ if ( parameters.modules ) {
+ for ( const k in parameters.modules ) {
+ modules[k] = parameters.modules[k];
+ }
+ }
+
+ instance.modules = modules;
+
+ // This "require" function can shadow the real one so
+ // that editor tools are aware of the modules that
+ // are being used.
+ instance.require = (name) => {
+ if ( modules[name] ) {
+ return modules[name];
+ }
+ return require(name);
+ }
+ },
+};
diff --git a/packages/puter-js-common/src/traits/PropertiesTrait.js b/packages/puter-js-common/src/traits/PropertiesTrait.js
new file mode 100644
index 00000000..7e35a9f3
--- /dev/null
+++ b/packages/puter-js-common/src/traits/PropertiesTrait.js
@@ -0,0 +1,20 @@
+module.exports = {
+ install_in_instance: (instance) => {
+ const properties = instance._get_merged_static_object('PROPERTIES');
+
+ for ( const k in properties ) {
+ if ( typeof properties[k] === 'function' ) {
+ instance[k] = properties[k]();
+ continue;
+ }
+
+ if ( typeof properties[k] === 'object' ) {
+ // This will be supported in the future.
+ throw new Error(`Property ${k} in ${instance.constructor.name} ` +
+ `is not a supported property specification.`);
+ }
+
+ instance[k] = properties[k];
+ }
+ }
+}
diff --git a/packages/puter-js-common/test/test.js b/packages/puter-js-common/test/test.js
new file mode 100644
index 00000000..ef08e5b4
--- /dev/null
+++ b/packages/puter-js-common/test/test.js
@@ -0,0 +1,54 @@
+const { expect } = require('chai');
+const { BasicBase } = require('../src/bases/BasicBase');
+const { AdvancedBase } = require('../src/AdvancedBase');
+
+class ClassA extends BasicBase {
+ static STATIC_OBJ = {
+ a: 1,
+ b: 2,
+ };
+ static STATIC_ARR = ['a', 'b'];
+}
+
+class ClassB extends ClassA {
+ static STATIC_OBJ = {
+ c: 3,
+ d: 4,
+ };
+ static STATIC_ARR = ['c', 'd'];
+}
+
+describe('testing', () => {
+ it('does a thing', () => {
+ const b = new ClassB();
+
+ console.log(b._get_inheritance_chain());
+ console.log([ClassA, ClassB]);
+ expect(b._get_inheritance_chain()).deep.equal([ClassA, ClassB]);
+ expect(b._get_merged_static_array('STATIC_ARR'))
+ .deep.equal(['a', 'b', 'c', 'd']);
+ expect(b._get_merged_static_object('STATIC_OBJ'))
+ .deep.equal({ a: 1, b: 2, c: 3, d: 4 });
+ });
+});
+
+class ClassWithModule extends AdvancedBase {
+ static MODULES = {
+ axios: 'axios',
+ };
+}
+
+describe('AdvancedBase', () => {
+ it('passes DI modules to instance', () => {
+ const c1 = new ClassWithModule();
+ expect(c1.modules.axios).to.equal('axios');
+
+ const c2 = new ClassWithModule({
+ modules: {
+ axios: 'my-axios',
+ },
+ });
+ expect(c2.modules.axios).to.equal('my-axios');
+ });
+});
+
diff --git a/puter-gui.json b/puter-gui.json
new file mode 100644
index 00000000..1d3579bc
--- /dev/null
+++ b/puter-gui.json
@@ -0,0 +1,31 @@
+{
+ "index": "/src/index.js",
+ "lib_paths": [
+ "/lib/jquery-3.6.1/jquery-3.6.1.min.js",
+ "/lib/viselect.min.js",
+ "/lib/FileSaver.min.js",
+ "/lib/socket.io/socket.io.min.js",
+ "/lib/qrcode.min.js",
+ "/lib/jquery-ui-1.13.2/jquery-ui.min.js",
+ "/lib/lodash@4.17.21.min.js",
+ "/lib/jquery.dragster.js",
+ "/lib/jquery.menu-aim.js",
+ "/lib/html-entities.js",
+ "/lib/timeago.min.js",
+ "/lib/iro.min.js",
+ "/lib/isMobile.min.js",
+ "/lib/jszip-3.10.1.min.js"
+ ],
+ "css_paths": [
+ "/css/normalize.css",
+ "/lib/jquery-ui-1.13.2/jquery-ui.min.css",
+ "/css/style.css"
+ ],
+ "js_paths": [
+ "/src/initgui.js",
+ "/src/helpers.js",
+ "/src/IPC.js",
+ "/src/globals.js",
+ "/src/i18n/i18n.js"
+ ]
+}
diff --git a/run-selfhosted.js b/run-selfhosted.js
new file mode 100644
index 00000000..af37f022
--- /dev/null
+++ b/run-selfhosted.js
@@ -0,0 +1,16 @@
+import backend from '@heyputer/backend';
+
+const {
+ Kernel,
+ CoreModule,
+ DatabaseModule,
+ PuterDriversModule,
+ LocalDiskStorageModule,
+} = backend;
+
+const k = new Kernel();
+k.add_module(new CoreModule());
+k.add_module(new DatabaseModule());
+k.add_module(new PuterDriversModule());
+k.add_module(new LocalDiskStorageModule());
+k.boot();
diff --git a/volatile/README.md b/volatile/README.md
new file mode 100644
index 00000000..a4421071
--- /dev/null
+++ b/volatile/README.md
@@ -0,0 +1,2 @@
+This directory contains ALL SAVED INFORMATION for Puter
+(unless you setup a production under `/etc` and `/var`)
diff --git a/volatile/config/.gitignore b/volatile/config/.gitignore
new file mode 100644
index 00000000..d6b7ef32
--- /dev/null
+++ b/volatile/config/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/volatile/runtime/.gitignore b/volatile/runtime/.gitignore
new file mode 100644
index 00000000..d6b7ef32
--- /dev/null
+++ b/volatile/runtime/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore