github_mirrors/puter
1
0
Fork 0
mirror of https://github.com/HeyPuter/puter.git synced 2025-02-02 23:28:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into add-context-menu-#876

Browse Source
This commit is contained in:
mariatouilzak 2024-12-18 10:43:47 +01:00 committed by GitHub
commit c7fb334f05
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
397 changed files with 17960 additions and 8596 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -26,3 +26,7 @@ dist/
# this is for jetbrain IDEs
.idea/
/puter
# Local Netlify folder
.netlify
src/emulator/release/

9
CONTRIBUTING.md
View File

@ -45,15 +45,18 @@ If you'd like to contribute code to Puter, you need to fork the project and subm
We'll review your pull request and work with you to get your changes merged into the project.
## Repository Structure
![file structure](./doc/File%20Structure.drawio.png)
## Your first code contribution
We maintain a list of issues that are good for first-time contributors. You can find these issues by searching for the [`good first issue`](https://github.com/HeyPuter/puter/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) label in our [GitHub repository](https://github.com/HeyPuter/puter). These issues are designed to be relatively easy to fix, and we're happy to help you get started. Pick an issue that interests you, and leave a comment on the issue to let us know you're working on it.
<br>
## Documentation for Contributors
See [doc/contributors/index.md](./doc/contributors/index.md) for more information.
### Backend
See [src/backend/CONTRIBUTING.md](src/backend/CONTRIBUTING.md)
<br>

1
awesome/#DoesItRunPuter.md
View File

@ -13,3 +13,4 @@
- [Steam Deck](https://twitter.com/everythingSung/status/1782162352403828793)
- [Ladybird Browser](https://x.com/HeyPuter/status/1810783504503800035)
- [Garry's Mod](https://x.com/HeyPuter/status/1850587712786722862)
- [Samsung Q88BA](https://x.com/AmirIsAround/status/1862614583263076540)

214
doc/File Structure.drawio Normal file
View File

File diff suppressed because one or more lines are too long

BIN
doc/File Structure.drawio.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 187 KiB

5
doc/contributors/extensions.md
View File

@ -1,2 +1,3 @@
### `vscode`
- `es6-string-html`
## Puter Extensions
See the [Wiki Page](https://github.com/HeyPuter/puter/wiki/ex_extensions)

2
doc/contributors/vscode.md Normal file
View File

@ -0,0 +1,2 @@
### `vscode`
- `es6-string-html`

9
doc/i18n/README.zh.md
View File

@ -89,6 +89,15 @@ docker compose up
```
<br/>
## 宝塔面板Docker一键部署（推荐）
1. 安装宝塔面板9.2.0及以上版本，前往 [宝塔面板](https://www.bt.cn/new/download.html?r=dk_puter) 官网，选择正式版的脚本下载安装
2. 安装后登录宝塔面板，在左侧菜单栏中点击 `Docker`，首次进入会提示安装`Docker`服务，点击立即安装，按提示完成安装
3. 安装完成后在应用商店中搜索`puter`，点击安装，配置域名等基本信息即可完成安装
### ☁️ Puter.com
Puter 可以作为托管服务使用，访问 [**puter.com**](https://puter.com)。

1
experiments/x86emu/www/main.js
View File

@ -17,7 +17,6 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#!/usr/bin/env node
/*
* Copyright (C) 2024 Puter Technologies Inc.
*

4
mods/mods_available/kdmod/ShareTestService.js
View File

@ -21,7 +21,6 @@
// we have these things registered in "useapi".
const {
get_user,
generate_system_fsentries,
invalidate_cached_user,
deleteUser,
} = require('../../../src/backend/src/helpers.js');
@ -146,7 +145,8 @@ class ShareTestService extends use.Service {
],
);
const user = await get_user({ username });
await generate_system_fsentries(user);
const svc_user = this.services.get('user');
await svc_user.generate_default_fsentries({ user });
invalidate_cached_user(user);
return user;
}

722
package-lock.json generated
View File

@ -17,6 +17,7 @@
"javascript-time-ago": "^2.5.11",
"json-colorizer": "^3.0.1",
"open": "^10.1.0",
"sharp": "^0.33.5",
"simple-git": "^3.25.0",
"string-template": "^1.0.0",
"uuid": "^9.0.1"
@ -1897,13 +1898,12 @@
}
},
"node_modules/@babel/code-frame": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz",
"integrity": "sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==",
"dev": true,
"license": "MIT",
"version": "7.26.2",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz",
"integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==",
"dependencies": {
"@babel/highlight": "^7.24.7",
"@babel/helper-validator-identifier": "^7.25.9",
"js-tokens": "^4.0.0",
"picocolors": "^1.0.0"
},
"engines": {
@ -1969,15 +1969,15 @@
}
},
"node_modules/@babel/generator": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.24.7.tgz",
"integrity": "sha512-oipXieGC3i45Y1A41t4tAqpnEZWgB/lC6Ehh6+rOviR5XWpTtMmLN+fGjz9vOiNRt0p6RtO6DtD0pdU3vpqdSA==",
"license": "MIT",
"version": "7.26.2",
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.2.tgz",
"integrity": "sha512-zevQbhbau95nkoxSq3f/DC/SC+EEOUZd3DYqfSkMhY2/wfSeaHV1Ew4vk8e+x8lja31IbyuUa2uQ3JONqKbysw==",
"dependencies": {
"@babel/types": "^7.24.7",
"@babel/parser": "^7.26.2",
"@babel/types": "^7.26.0",
"@jridgewell/gen-mapping": "^0.3.5",
"@jridgewell/trace-mapping": "^0.3.25",
"jsesc": "^2.5.1"
"jsesc": "^3.0.2"
},
"engines": {
"node": ">=6.9.0"
@ -2023,33 +2023,6 @@
"node": ">=6.9.0"
}
},
"node_modules/@babel/helper-function-name": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.24.7.tgz",
"integrity": "sha512-FyoJTsj/PEUWu1/TYRiXTIHc8lbw+TDYkZuoE43opPS5TrI7MyONBE1oNvfguEXAD9yhQRrVBnXdXzSLQl9XnA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/template": "^7.24.7",
"@babel/types": "^7.24.7"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/helper-hoist-variables": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.24.7.tgz",
"integrity": "sha512-MJJwhkoGy5c4ehfoRyrJ/owKeMl19U54h27YYftT0o2teQ3FJ3nQUf/I3LlJsX4l3qlw7WRXUmiyajvHXoTubQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/types": "^7.24.7"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/helper-module-imports": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz",
@ -2112,19 +2085,17 @@
}
},
"node_modules/@babel/helper-string-parser": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.7.tgz",
"integrity": "sha512-7MbVt6xrwFQbunH2DNQsAP5sTGxfqQtErvBIvIMi6EQnbgUOuVYanvREcmFrOPhoXBrTtjhhP+lW+o5UfK+tDg==",
"license": "MIT",
"version": "7.25.9",
"resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz",
"integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==",
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/helper-validator-identifier": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz",
"integrity": "sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==",
"license": "MIT",
"version": "7.25.9",
"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz",
"integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==",
"engines": {
"node": ">=6.9.0"
}
@ -2153,82 +2124,13 @@
"node": ">=6.9.0"
}
},
"node_modules/@babel/highlight": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.7.tgz",
"integrity": "sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/helper-validator-identifier": "^7.24.7",
"chalk": "^2.4.2",
"js-tokens": "^4.0.0",
"picocolors": "^1.0.0"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/highlight/node_modules/ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"dev": true,
"license": "MIT",
"dependencies": {
"color-convert": "^1.9.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/@babel/highlight/node_modules/chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"ansi-styles": "^3.2.1",
"escape-string-regexp": "^1.0.5",
"supports-color": "^5.3.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/@babel/highlight/node_modules/color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"dev": true,
"license": "MIT",
"dependencies": {
"color-name": "1.1.3"
}
},
"node_modules/@babel/highlight/node_modules/color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
"dev": true,
"license": "MIT"
},
"node_modules/@babel/highlight/node_modules/escape-string-regexp": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.8.0"
}
},
"node_modules/@babel/parser": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.7.tgz",
"integrity": "sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw==",
"license": "MIT",
"version": "7.26.2",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.2.tgz",
"integrity": "sha512-DWMCZH9WA4Maitz2q21SRKHo9QXZxkDsbNZoVD62gusNtNBBqDg9i7uOhASfTfIGNzW+O+r7+jAlM8dwphcJKQ==",
"dependencies": {
"@babel/types": "^7.26.0"
},
"bin": {
"parser": "bin/babel-parser.js"
},
@ -2250,35 +2152,28 @@
}
},
"node_modules/@babel/template": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.7.tgz",
"integrity": "sha512-jYqfPrU9JTF0PmPy1tLYHW4Mp4KlgxJD9l2nP9fD6yT/ICi554DmrWBAEYpIelzjHf1msDP3PxJIRt/nFNfBig==",
"dev": true,
"license": "MIT",
"version": "7.25.9",
"resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz",
"integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==",
"dependencies": {
"@babel/code-frame": "^7.24.7",
"@babel/parser": "^7.24.7",
"@babel/types": "^7.24.7"
"@babel/code-frame": "^7.25.9",
"@babel/parser": "^7.25.9",
"@babel/types": "^7.25.9"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/traverse": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.24.7.tgz",
"integrity": "sha512-yb65Ed5S/QAcewNPh0nZczy9JdYXkkAbIsEo+P7BE7yO3txAY30Y/oPa3QkQ5It3xVG2kpKMg9MsdxZaO31uKA==",
"dev": true,
"license": "MIT",
"version": "7.25.9",
"resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.9.tgz",
"integrity": "sha512-ZCuvfwOwlz/bawvAuvcj8rrithP2/N55Tzz342AkTvq4qaWbGfmCk/tKhNaV2cthijKrPAA8SRJV5WWe7IBMJw==",
"dependencies": {
"@babel/code-frame": "^7.24.7",
"@babel/generator": "^7.24.7",
"@babel/helper-environment-visitor": "^7.24.7",
"@babel/helper-function-name": "^7.24.7",
"@babel/helper-hoist-variables": "^7.24.7",
"@babel/helper-split-export-declaration": "^7.24.7",
"@babel/parser": "^7.24.7",
"@babel/types": "^7.24.7",
"@babel/code-frame": "^7.25.9",
"@babel/generator": "^7.25.9",
"@babel/parser": "^7.25.9",
"@babel/template": "^7.25.9",
"@babel/types": "^7.25.9",
"debug": "^4.3.1",
"globals": "^11.1.0"
},
@ -2290,21 +2185,18 @@
"version": "11.12.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
"integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/@babel/types": {
"version": "7.24.7",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.7.tgz",
"integrity": "sha512-XEFXSlxiG5td2EJRe8vOmRbaXVgfcBlszKujvVmWIK/UpywWljQCfzAv3RQCGujWQ1RD4YYWEAqDXfuJiy8f5Q==",
"license": "MIT",
"version": "7.26.0",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz",
"integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==",
"dependencies": {
"@babel/helper-string-parser": "^7.24.7",
"@babel/helper-validator-identifier": "^7.24.7",
"to-fast-properties": "^2.0.0"
"@babel/helper-string-parser": "^7.25.9",
"@babel/helper-validator-identifier": "^7.25.9"
},
"engines": {
"node": ">=6.9.0"
@ -2339,6 +2231,15 @@
"node": ">=10.0.0"
}
},
"node_modules/@emnapi/runtime": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz",
"integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==",
"optional": true,
"dependencies": {
"tslib": "^2.4.0"
}
},
"node_modules/@eslint-community/eslint-utils": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
@ -2596,8 +2497,8 @@
"form-data": "^4.0.0"
}
},
"node_modules/@heyputer/parsely": {
"resolved": "src/parsely",
"node_modules/@heyputer/parsers": {
"resolved": "src/parsers",
"link": true
},
"node_modules/@heyputer/phoenix": {
@ -2666,6 +2567,348 @@
"url": "https://github.com/sponsors/nzakas"
}
},
"node_modules/@img/sharp-darwin-arm64": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz",
"integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-darwin-arm64": "1.0.4"
}
},
"node_modules/@img/sharp-darwin-x64": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz",
"integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-darwin-x64": "1.0.4"
}
},
"node_modules/@img/sharp-libvips-darwin-arm64": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz",
"integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"darwin"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-darwin-x64": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz",
"integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"darwin"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linux-arm": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz",
"integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==",
"cpu": [
"arm"
],
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linux-arm64": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz",
"integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linux-s390x": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz",
"integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==",
"cpu": [
"s390x"
],
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linux-x64": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz",
"integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linuxmusl-arm64": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz",
"integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linuxmusl-x64": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz",
"integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-linux-arm": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz",
"integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==",
"cpu": [
"arm"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linux-arm": "1.0.5"
}
},
"node_modules/@img/sharp-linux-arm64": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz",
"integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linux-arm64": "1.0.4"
}
},
"node_modules/@img/sharp-linux-s390x": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz",
"integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==",
"cpu": [
"s390x"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linux-s390x": "1.0.4"
}
},
"node_modules/@img/sharp-linux-x64": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz",
"integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linux-x64": "1.0.4"
}
},
"node_modules/@img/sharp-linuxmusl-arm64": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz",
"integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linuxmusl-arm64": "1.0.4"
}
},
"node_modules/@img/sharp-linuxmusl-x64": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz",
"integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linuxmusl-x64": "1.0.4"
}
},
"node_modules/@img/sharp-wasm32": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz",
"integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==",
"cpu": [
"wasm32"
],
"optional": true,
"dependencies": {
"@emnapi/runtime": "^1.2.0"
},
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-win32-ia32": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz",
"integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==",
"cpu": [
"ia32"
],
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-win32-x64": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz",
"integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@istanbuljs/load-nyc-config": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz",
@ -7800,9 +8043,9 @@
}
},
"node_modules/axios": {
"version": "1.7.7",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz",
"integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==",
"version": "1.7.8",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.7.8.tgz",
"integrity": "sha512-Uu0wb7KNqK2t5K+YQyVCLM76prD5sRFjKHbJYCP1J7JFGEQ6nN7HWn9+04LAeiJ3ji54lgS/gZCH1oxyrf1SPw==",
"dependencies": {
"follow-redirects": "^1.15.6",
"form-data": "^4.0.0",
@ -8634,6 +8877,10 @@
"resolved": "tools/comment-parser",
"link": true
},
"node_modules/comment-writer": {
"resolved": "tools/comment-writer",
"link": true
},
"node_modules/commondir": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
@ -8843,10 +9090,6 @@
"node": ">= 0.6"
}
},
"node_modules/contextlink": {
"resolved": "src/contextlink",
"link": true
},
"node_modules/convert-source-map": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz",
@ -9215,7 +9458,6 @@
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz",
"integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==",
"license": "MIT",
"peerDependencies": {
"babel-plugin-macros": "^3.1.0"
},
@ -9422,6 +9664,17 @@
"node": ">=8"
}
},
"node_modules/doctrine": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
"integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
"dependencies": {
"esutils": "^2.0.2"
},
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/dom-converter": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz",
@ -9624,7 +9877,6 @@
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz",
"integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==",
"license": "MIT",
"dependencies": {
"ansi-colors": "^4.1.1",
"strip-ansi": "^6.0.1"
@ -9883,7 +10135,6 @@
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
"integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
"license": "BSD-2-Clause",
"bin": {
"esparse": "bin/esparse.js",
"esvalidate": "bin/esvalidate.js"
@ -12045,9 +12296,7 @@
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"dev": true,
"license": "MIT"
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="
},
"node_modules/js-yaml": {
"version": "4.1.0",
@ -12062,15 +12311,14 @@
}
},
"node_modules/jsesc": {
"version": "2.5.2",
"resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",
"integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==",
"license": "MIT",
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz",
"integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==",
"bin": {
"jsesc": "bin/jsesc"
},
"engines": {
"node": ">=4"
"node": ">=6"
}
},
"node_modules/json-bigint": {
@ -13120,6 +13368,10 @@
"integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==",
"license": "MIT"
},
"node_modules/module-docgen": {
"resolved": "tools/module-docgen",
"link": true
},
"node_modules/moment": {
"version": "2.30.1",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
@ -15089,12 +15341,9 @@
"license": "MIT"
},
"node_modules/semver": {
"version": "7.6.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz",
"integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==",
"dependencies": {
"lru-cache": "^6.0.0"
},
"version": "7.6.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
"integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
"bin": {
"semver": "bin/semver.js"
},
@ -15102,22 +15351,6 @@
"node": ">=10"
}
},
"node_modules/semver/node_modules/lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dependencies": {
"yallist": "^4.0.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/semver/node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node_modules/send": {
"version": "0.19.0",
"resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
@ -15242,6 +15475,56 @@
"node": ">=8"
}
},
"node_modules/sharp": {
"version": "0.33.5",
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz",
"integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==",
"hasInstallScript": true,
"dependencies": {
"color": "^4.2.3",
"detect-libc": "^2.0.3",
"semver": "^7.6.3"
},
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-darwin-arm64": "0.33.5",
"@img/sharp-darwin-x64": "0.33.5",
"@img/sharp-libvips-darwin-arm64": "1.0.4",
"@img/sharp-libvips-darwin-x64": "1.0.4",
"@img/sharp-libvips-linux-arm": "1.0.5",
"@img/sharp-libvips-linux-arm64": "1.0.4",
"@img/sharp-libvips-linux-s390x": "1.0.4",
"@img/sharp-libvips-linux-x64": "1.0.4",
"@img/sharp-libvips-linuxmusl-arm64": "1.0.4",
"@img/sharp-libvips-linuxmusl-x64": "1.0.4",
"@img/sharp-linux-arm": "0.33.5",
"@img/sharp-linux-arm64": "0.33.5",
"@img/sharp-linux-s390x": "0.33.5",
"@img/sharp-linux-x64": "0.33.5",
"@img/sharp-linuxmusl-arm64": "0.33.5",
"@img/sharp-linuxmusl-x64": "0.33.5",
"@img/sharp-wasm32": "0.33.5",
"@img/sharp-win32-ia32": "0.33.5",
"@img/sharp-win32-x64": "0.33.5"
}
},
"node_modules/sharp/node_modules/color": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
"integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
"dependencies": {
"color-convert": "^2.0.1",
"color-string": "^1.9.0"
},
"engines": {
"node": ">=12.5.0"
}
},
"node_modules/shebang-command": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@ -15612,10 +15895,6 @@
"node": ">= 0.8"
}
},
"node_modules/strataparse": {
"resolved": "src/strataparse",
"link": true
},
"node_modules/streamsearch": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
@ -16052,15 +16331,6 @@
"integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==",
"license": "MIT"
},
"node_modules/to-fast-properties": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
"integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==",
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@ -16842,6 +17112,14 @@
"integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==",
"license": "MIT"
},
"node_modules/word-wrap": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
"integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/wordwrap": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
@ -17412,6 +17690,7 @@
},
"src/contextlink": {
"version": "0.0.0",
"extraneous": true,
"license": "AGPL-3.0-only",
"devDependencies": {
"mocha": "^10.2.0"
@ -17489,6 +17768,12 @@
"src/parsely": {
"name": "@heyputer/parsely",
"version": "1.0.0",
"extraneous": true,
"license": "AGPL-3.0-only"
},
"src/parsers": {
"name": "@heyputer/parsers",
"version": "1.0.0",
"license": "AGPL-3.0-only"
},
"src/phoenix": {
@ -17594,6 +17879,7 @@
},
"src/strataparse": {
"version": "0.0.0",
"extraneous": true,
"license": "AGPL-3.0-only"
},
"src/terminal": {
@ -17693,6 +17979,20 @@
"node": ">= 14.16"
}
},
"tools/comment-writer": {
"version": "1.0.0",
"license": "AGPL-3.0-only",
"dependencies": {
"axios": "^1.7.8",
"console-table-printer": "^2.12.1",
"dedent": "^1.5.3",
"diff-match-patch": "^1.0.5",
"enquirer": "^2.4.1",
"js-levenshtein": "^1.1.6",
"word-wrap": "^1.2.5",
"yaml": "^2.4.5"
}
},
"tools/file-walker": {
"version": "1.0.0",
"license": "AGPL-3.0-only"
@ -17738,6 +18038,16 @@
"node": ">=18"
}
},
"tools/module-docgen": {
"version": "1.0.0",
"license": "AGPL-3.0-only",
"dependencies": {
"@babel/parser": "^7.26.2",
"@babel/traverse": "^7.25.9",
"dedent": "^1.5.3",
"doctrine": "^3.0.0"
}
},
"tools/token-count-accuracy": {
"version": "1.0.0",
"license": "AGPL-3.0-only"

3
package.json
View File

@ -28,7 +28,7 @@
"webpack-cli": "^5.1.1"
},
"scripts": {
"test": "npx mocha src/phoenix/test src/contextlink/test && node src/backend/tools/test",
"test": "npx mocha src/phoenix/test && node src/backend/tools/test",
"start=gui": "nodemon --exec \"node dev-server.js\" ",
"start": "node ./tools/run-selfhosted.js",
"build": "cd src/gui; node ./build.js",
@ -50,6 +50,7 @@
"javascript-time-ago": "^2.5.11",
"json-colorizer": "^3.0.1",
"open": "^10.1.0",
"sharp": "^0.33.5",
"simple-git": "^3.25.0",
"string-template": "^1.0.0",
"uuid": "^9.0.1"

34
src/backend/doc/contributors/index.md → src/backend/CONTRIBUTING.md
View File

@ -1,18 +1,22 @@
# Contributing to Puter's Backend
## File Structure
## Architecture
- [boot sequence](./boot-sequence.md)
- [modules and services](./modules.md)
- [boot sequence](./doc/contributors/boot-sequence.md)
- [modules and services](./doc/contributors/modules.md)
## Features
- [protected apps](../features/protected-apps.md)
- [service scripts](../features/service-scripts.md)
- [protected apps](./doc/features/protected-apps.md)
- [service scripts](./doc/features/service-scripts.md)
## Lists of Things
- [list of permissions](../lists-of-things/list-of-permissions.md)
- [list of permissions](./doc/lists-of-things/list-of-permissions.md)
## Code-First Approach
@ -20,21 +24,21 @@ If you prefer to understand a system by looking at the
first files which are invoked and starting from there,
here's a handy list!
- [Kernel](../../src/Kernel.js), despite its intimidating name, is a
- [Kernel](./src/Kernel.js), despite its intimidating name, is a
relatively simple (< 200 LOC) class which loads the modules
(modules register services), and then starts all the services.
- [RuntimeEnvironment](../../src/boot/RuntimeEnvironment.js)
- [RuntimeEnvironment](./src/boot/RuntimeEnvironment.js)
sets the configuration and runtime directories. It's invoked by Kernel.
- The default setup for running a self-hosted Puter loads these modules:
- [CoreModule](../../src/CoreModule.js)
- [DatabaseModule](../../src/DatabaseModule.js)
- [LocalDiskStorageModule](../../src/LocalDiskStorageModule.js)
- [CoreModule](./src/CoreModule.js)
- [DatabaseModule](./src/DatabaseModule.js)
- [LocalDiskStorageModule](./src/LocalDiskStorageModule.js)
- HTTP endpoints are registered with
[WebServerService](../../src/services/WebServerService.js)
[WebServerService](./src/services/WebServerService.js)
by these services:
- [ServeGUIService](../../src/services/ServeGUIService.js)
- [PuterAPIService](../../src/services/PuterAPIService.js)
- [FilesystemAPIService](../../src/services/FilesystemAPIService.js)
- [ServeGUIService](./src/services/ServeGUIService.js)
- [PuterAPIService](./src/services/PuterAPIService.js)
- [FilesystemAPIService](./src/services/FilesystemAPIService.js)
## Development Philosophies
@ -71,7 +75,7 @@ doing the useless work that reveals what the useful work is.
## Underlying Constructs
- [putility's README.md](../../packages/putility/README.md)
- [putility's README.md](../putility/README.md)
- Whenever you see `AdvancedBase`, that's from here
- Many things in backend extend this. Anything that doesn't only doesn't
because it was written before `AdvancedBase` existed.

65
src/backend/doc/Kernel.md Normal file
View File

@ -0,0 +1,65 @@
# Puter Kernel Documentation
## Overview
The **Puter Kernel** is the core runtime component of the Puter system. It provides the foundational infrastructure for:
- Initializing the runtime environment
- Managing internal and external modules (extensions)
- Setting up and booting core services
- Configuring logging and debugging utilities
- Integrating with third-party modules and performing dependency installs at runtime
This kernel is responsible for orchestrating the startup sequence and ensuring that all necessary services, modules, and environmental configurations are properly loaded before the application enters its operational state.
---
## Features
1. **Modular Architecture**:
The Kernel supports both internal and external modules:
- **Internal Modules**: Provided to Kernel by an initializing script, such
as `tools/run-selfhosted.js`, via the `add_module()` method.
- **External Modules**: Discovered in configured module directories and installed
dynamically. This includes resolving and executing `package.json` entries and
running `npm install` as needed.
2. **Service Container & Registry**:
The Kernel initializes a service container that manages a wide range of services. Services can:
- Register modules
- Initialize dependencies
- Emit lifecycle events (`boot.consolidation`, `boot.activation`, `boot.ready`) to
orchestrate a stable and consistent environment.
3. **Runtime Environment Setup**:
The Kernel sets up a `RuntimeEnvironment` to determine configuration paths and environment parameters. It also provides global helpers like `kv` for key-value storage and `cl` for simplified console logging.
4. **Logging and Debugging**:
Uses a temporary `BootLogger` for the initialization phase until LogService is
initialized, at which point it will replace the boot logger. Debugging features
(`ll`, `xtra_log`) are enabled in development environments for convenience.
## Initialization & Boot Process
1. **Constructor**:
When a Kernel instance is created, it sets up basic parameters, initializes an empty
module list, and prepares `useapi()` integration.
2. **Booting**:
The `boot()` method:
- Parses CLI arguments using `yargs`.
- Calls `_runtime_init()` to set up the `RuntimeEnvironment` and boot logger.
- Initializes global debugging/logging utilities.
- Sets up the service container (usually called `services`c instance of **Container**).
- Invokes module installation and service bootstrapping processes.
3. **Module Installation**:
Internal modules are registered and installed first.
External modules are discovered, packaged, installed, and their code is executed.
External modules are given a special context with access to `useapi()`, a dynamic
import mechanism for Puter modules and extensions.
4. **Service Bootstrapping**:
After modules and extensions are installed, services are initialized and activated.
For more information about how this works, see [boot-sequence.md](./contributors/boot-sequence.md).

19
src/backend/exports.js
View File

@ -21,13 +21,17 @@ const { Kernel } = require("./src/Kernel.js");
const DatabaseModule = require("./src/DatabaseModule.js");
const LocalDiskStorageModule = require("./src/LocalDiskStorageModule.js");
const SelfHostedModule = require("./src/modules/selfhosted/SelfHostedModule.js");
const PuterDriversModule = require("./src/PuterDriversModule.js");
const { testlaunch } = require("./src/index.js");
const BaseService = require("./src/services/BaseService.js");
const { Context } = require("./src/util/context.js");
const { TestDriversModule } = require("./src/modules/test-drivers/TestDriversModule.js");
const { PuterAIModule } = require("./src/modules/puterai/PuterAIModule.js");
const { BroadcastModule } = require("./src/modules/broadcast/BroadcastModule.js");
const { WebModule } = require("./src/modules/web/WebModule.js");
const { Core2Module } = require("./src/modules/core/Core2Module.js");
const { TemplateModule } = require("./src/modules/template/TemplateModule.js");
const { PuterFSModule } = require("./src/modules/puterfs/PuterFSModule.js");
const { PerfMonModule } = require("./src/modules/perfmon/PerfMonModule.js");
module.exports = {
@ -43,13 +47,24 @@ module.exports = {
Kernel,
EssentialModules: [
Core2Module,
PuterFSModule,
CoreModule,
WebModule,
TemplateModule,
],
// Pre-built modules
CoreModule,
WebModule,
DatabaseModule,
PuterDriversModule,
LocalDiskStorageModule,
SelfHostedModule,
TestDriversModule,
PuterAIModule,
BroadcastModule,
// Development modules
PerfMonModule,
};

67
src/backend/src/CoreModule.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"claude"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@ -23,6 +24,16 @@ const { ProtectedAppES } = require("./om/entitystorage/ProtectedAppES");
const { Context } = require('./util/context');
/**
* Core module for the Puter platform that includes essential services including
* authentication, filesystems, rate limiting, permissions, and various API endpoints.
*
* This is a monolithic module. Incrementally, services should be migrated to
* Core2Module and other modules instead. Core2Module has a smaller scope, and each
* new module will be a cohesive concern. Once CoreModule is empty, it will be removed
* and Core2Module will take on its name.
*/
class CoreModule extends AdvancedBase {
dirname () { return __dirname; }
async install (context) {
@ -33,11 +44,16 @@ class CoreModule extends AdvancedBase {
await install({ services, app, useapi, modapi });
}
// Some services were created before the BaseService
// class existed. They don't listen to the init event
// and the order in which they're instantiated matters.
// They all need to be installed after the init event
// is dispatched, so they get a separate install method.
/**
* Installs legacy services that don't extend BaseService and require special handling.
* These services were created before the BaseService class existed and don't listen
* to the init event. They need to be installed after the init event is dispatched
* due to initialization order dependencies.
*
* @param {Object} context - The context object containing service references
* @param {Object} context.services - Service registry for registering legacy services
* @returns {Promise<void>} Resolves when legacy services are installed
*/
async install_legacy (context) {
const services = context.get('services');
await install_legacy({ services });
@ -52,6 +68,9 @@ module.exports = CoreModule;
const install = async ({ services, app, useapi, modapi }) => {
const config = require('./config');
// === LIBRARIES ===
useapi.withuse(() => {
def('Service', require('./services/BaseService'));
def('Module', AdvancedBase);
@ -68,7 +87,6 @@ const install = async ({ services, app, useapi, modapi }) => {
def('core.config', config);
});
// === LIBRARIES ===
useapi.withuse(() => {
const ArrayUtil = require('./libraries/ArrayUtil');
services.registerService('util-array', ArrayUtil);
@ -82,16 +100,11 @@ const install = async ({ services, app, useapi, modapi }) => {
// === SERVICES ===
// /!\ IMPORTANT /!\
// For new services, put the import immediate above the
// For new services, put the import immediately above the
// call to services.registerService. We'll clean this up
// in a future PR.
const { LogService } = require('./services/runtime-analysis/LogService');
const { PagerService } = require('./services/runtime-analysis/PagerService');
const { AlarmService } = require('./services/runtime-analysis/AlarmService');
const { ErrorService } = require('./services/runtime-analysis/ErrorService');
const { CommandService } = require('./services/CommandService');
const { ExpectationService } = require('./services/runtime-analysis/ExpectationService');
const { HTTPThumbnailService } = require('./services/thumbnails/HTTPThumbnailService');
const { PureJSThumbnailService } = require('./services/thumbnails/PureJSThumbnailService');
const { NAPIThumbnailService } = require('./services/thumbnails/NAPIThumbnailService');
@ -124,12 +137,10 @@ const install = async ({ services, app, useapi, modapi }) => {
const { ESBuilder } = require('./om/entitystorage/ESBuilder');
const { Eq, Or } = require('./om/query/query');
const { TrackSpendingService } = require('./services/TrackSpendingService');
const { ServerHealthService } = require('./services/runtime-analysis/ServerHealthService');
const { MakeProdDebuggingLessAwfulService } = require('./services/MakeProdDebuggingLessAwfulService');
const { ConfigurableCountingService } = require('./services/ConfigurableCountingService');
const { FSLockService } = require('./services/fs/FSLockService');
const { StrategizedService } = require('./services/StrategizedService');
const WebServerService = require('./services/WebServerService');
const FilesystemAPIService = require('./services/FilesystemAPIService');
const ServeGUIService = require('./services/ServeGUIService');
const PuterAPIService = require('./services/PuterAPIService');
@ -140,17 +151,10 @@ const install = async ({ services, app, useapi, modapi }) => {
// === Services which extend BaseService ===
services.registerService('system-validation', SystemValidationService);
services.registerService('server-health', ServerHealthService);
services.registerService('log-service', LogService);
services.registerService('commands', CommandService);
services.registerService('web-server', WebServerService, { app });
services.registerService('__api-filesystem', FilesystemAPIService);
services.registerService('__api', PuterAPIService);
services.registerService('__gui', ServeGUIService);
services.registerService('expectations', ExpectationService);
services.registerService('pager', PagerService);
services.registerService('alarm', AlarmService);
services.registerService('error-service', ErrorService);
services.registerService('registry', RegistryService);
services.registerService('__registrant', RegistrantService);
services.registerService('fslock', FSLockService);
@ -187,9 +191,6 @@ const install = async ({ services, app, useapi, modapi }) => {
]),
});
const { ParameterService } = require('./services/ParameterService');
services.registerService('params', ParameterService);
const { InformationService } = require('./services/information/InformationService');
services.registerService('information', InformationService)
@ -316,9 +317,6 @@ const install = async ({ services, app, useapi, modapi }) => {
const { PermissionAPIService } = require('./services/PermissionAPIService');
services.registerService('__permission-api', PermissionAPIService);
const { MountpointService } = require('./services/MountpointService');
services.registerService('mountpoint', MountpointService);
const { AnomalyService } = require('./services/AnomalyService');
services.registerService('anomaly', AnomalyService);
@ -351,29 +349,32 @@ const install = async ({ services, app, useapi, modapi }) => {
const { ReferralCodeService } = require('./services/ReferralCodeService');
services.registerService('referral-code', ReferralCodeService);
const { UserService } = require('./services/UserService');
services.registerService('user', UserService);
const { WSPushService } = require('./services/WSPushService');
services.registerService('__event-push-ws', WSPushService);
const { AppIconService } = require('./services/AppIconService');
services.registerService('app-icon', AppIconService);
}
const install_legacy = async ({ services }) => {
const { ProcessEventService } = require('./services/runtime-analysis/ProcessEventService');
// const { FilesystemService } = require('./filesystem/FilesystemService');
const PerformanceMonitor = require('./monitor/PerformanceMonitor');
const { OperationTraceService } = require('./services/OperationTraceService');
const { WSPushService } = require('./services/WSPushService');
const { ClientOperationService } = require('./services/ClientOperationService');
const { EngPortalService } = require('./services/EngPortalService');
const { AppInformationService } = require('./services/AppInformationService');
const { FileCacheService } = require('./services/file-cache/FileCacheService');
// === Services which do not yet extend BaseService ===
services.registerService('process-event', ProcessEventService);
// services.registerService('filesystem', FilesystemService);
services.registerService('operationTrace', OperationTraceService);
services.registerService('__event-push-ws', WSPushService);
services.registerService('file-cache', FileCacheService);
services.registerService('client-operation', ClientOperationService);
services.registerService('app-information', AppInformationService);
services.registerService('engineering-portal', EngPortalService);
// TODO: add to here: ResourceService and DatabaseFSEntryService
// This singleton was made before services existed,
// so we have to pass that to it manually

26
src/backend/src/Extension.js
View File

@ -3,6 +3,10 @@ const EmitterFeature = require("@heyputer/putility/src/features/EmitterFeature")
const { Context } = require("./util/context");
const { ExtensionServiceState } = require("./ExtensionService");
/**
* This class creates the `extension` global that is seem by Puter backend
* extensions.
*/
class Extension extends AdvancedBase {
static FEATURES = [
EmitterFeature({
@ -24,6 +28,9 @@ class Extension extends AdvancedBase {
console.log('Example method called by an extension.');
}
/**
* This will get a database instance from the default service.
*/
get db () {
const db = this.service.values.get('db');
if ( ! db ) {
@ -35,6 +42,12 @@ class Extension extends AdvancedBase {
return db;
}
/**
* This will create a GET endpoint on the default service.
* @param {*} path - route for the endpoint
* @param {*} handler - function to handle the endpoint
* @param {*} options - options like noauth (bool) and mw (array)
*/
get (path, handler, options) {
// this extension will have a default service
this.ensure_service_();
@ -51,6 +64,12 @@ class Extension extends AdvancedBase {
});
}
/**
* This will create a POST endpoint on the default service.
* @param {*} path - route for the endpoint
* @param {*} handler - function to handle the endpoint
* @param {*} options - options like noauth (bool) and mw (array)
*/
post (path, handler, options) {
// this extension will have a default service
this.ensure_service_();
@ -67,6 +86,13 @@ class Extension extends AdvancedBase {
});
}
/**
* This method will create the "default service" for an extension.
* This is specifically for Puter extensions that do not define their
* own service classes.
*
* @returns {void}
*/
ensure_service_ () {
if ( this.service ) {
return;

6
src/backend/src/ExtensionService.js
View File

@ -5,6 +5,11 @@ const configurable_auth = require("./middleware/configurable_auth");
const { Context } = require("./util/context");
const { DB_READ, DB_WRITE } = require("./services/database/consts");
/**
* State shared with the default service and the `extension` global so that
* methods on `extension` can register routes (and make other changes in the
* future) to the default service.
*/
class ExtensionServiceState extends AdvancedBase {
constructor (...a) {
super(...a);
@ -75,6 +80,7 @@ class ExtensionService extends BaseService {
}
['__on_install.routes'] (_, { app }) {
if ( ! this.state ) debugger;
for ( const endpoint of this.state.endpoints_ ) {
endpoint.attach(app);
}

116
src/backend/src/Kernel.js
View File

@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
const { AdvancedBase, libs } = require("@heyputer/putility");
const { Context } = require('./util/context');
const BaseService = require("./services/BaseService");
const useapi = require('useapi');
@ -25,7 +25,8 @@ const { hideBin } = require('yargs/helpers');
const { Extension } = require("./Extension");
const { ExtensionModule } = require("./ExtensionModule");
const { spawn } = require("node:child_process");
const { quot } = require("./util/strutil");
const { quot } = libs.string;
class Kernel extends AdvancedBase {
constructor ({ entry_path } = {}) {
@ -78,8 +79,6 @@ class Kernel extends AdvancedBase {
this._runtime_init({ args });
// const express = require('express')
// const app = express();
const config = require('./config');
globalThis.ll = o => o;
@ -101,18 +100,11 @@ class Kernel extends AdvancedBase {
const { consoleLogManager } = require('./util/consolelog');
consoleLogManager.initialize_proxy_methods();
// TODO: temporary dependency inversion; requires moving:
// - rm, so we can move mv
// - mv, so we can move mkdir
// - generate_default_fsentries, so we can move mkdir
// - mkdir, which needs an fs provider
// === START: Initialize Service Registry ===
const { Container } = require('./services/Container');
const services = new Container({ logger: this.bootLogger });
this.services = services;
// app.set('services', services);
const root_context = Context.create({
environment: this.environment,
@ -130,7 +122,6 @@ class Kernel extends AdvancedBase {
});
// Error.stackTraceLimit = Infinity;
Error.stackTraceLimit = 200;
}
@ -238,7 +229,22 @@ class Kernel extends AdvancedBase {
}
const mod_dirnames = fs.readdirSync(mods_dirpath);
for ( const mod_dirname of mod_dirnames ) {
let mod_path = path_.join(mods_dirpath, mod_dirname);
await this.install_extern_mod_({
mod_install_root_context,
mod_dirname,
mod_path: path_.join(mods_dirpath, mod_dirname),
});
}
}
}
async install_extern_mod_({
mod_install_root_context,
mod_dirname,
mod_path,
}) {
const path_ = require('path');
const fs = require('fs');
let stat = fs.lstatSync(mod_path);
while ( stat.isSymbolicLink() ) {
@ -246,8 +252,9 @@ class Kernel extends AdvancedBase {
stat = fs.lstatSync(mod_path);
}
if ( ! stat.isDirectory() && !(mod_dirname.endsWith('.js')) ) {
continue;
// Mod must be a directory or javascript file
if ( ! stat.isDirectory() && !(mod_path.endsWith('.js')) ) {
return;
}
const mod_name = path_.parse(mod_path).name;
@ -257,41 +264,24 @@ class Kernel extends AdvancedBase {
if ( ! stat.isDirectory() ) {
this.create_mod_package_json(mod_package_dir, {
name: mod_name,
entry: 'main.js'
});
fs.copyFileSync(mod_path, path_.join(mod_package_dir, 'main.js'));
} else {
if ( ! fs.existsSync(path_.join(mod_path, 'package.json')) ) {
// Expect main.js or index.js to exist
const options = ['main.js', 'index.js'];
let entry_file = null;
for ( const option of options ) {
if ( fs.existsSync(path_.join(mod_path, option)) ) {
entry_file = option;
break;
}
}
if ( ! entry_file ) {
// If directory is empty, we'll just skip it
if ( fs.readdirSync(mod_path).length === 0 ) {
this.bootLogger.warn(`Empty mod directory ${quot(mod_path)}; skipping...`);
continue;
}
// Other wise, we'll throw an error
this.bootLogger.error(`Expected main.js or index.js in ${quot(mod_path)}`);
if ( ! process.env.SKIP_INVALID_MODS ) {
this.bootLogger.error(`Set SKIP_INVALID_MODS=1 (environment variable) to run anyway.`);
process.exit(1);
} else {
continue;
}
return;
}
// Create package.json if it doesn't exist
if ( ! fs.existsSync(path_.join(mod_path, 'package.json')) ) {
this.create_mod_package_json(mod_package_dir, {
name: mod_name,
entry: entry_file,
});
}
// Copy mod contents to `/mod_packages`
fs.cpSync(mod_path, mod_package_dir, {
recursive: true,
});
@ -304,6 +294,13 @@ class Kernel extends AdvancedBase {
const mod = new ExtensionModule();
mod.extension = new Extension();
const mod_context = this._create_mod_context(mod_install_root_context, {
name: mod_dirname,
['module']: mod,
external: true,
mod_path,
});
// This is where the module gets the 'use' and 'def' globals
await this.useapi.awithuse(async () => {
// This is where the module gets the 'extension' global
@ -314,28 +311,11 @@ class Kernel extends AdvancedBase {
if ( maybe_promise && maybe_promise instanceof Promise ) {
await maybe_promise;
}
});
});
const mod_context = this._create_mod_context(mod_install_root_context, {
name: mod_dirname,
['module']: mod,
external: true,
mod_path,
});
// TODO: DRY `awithuse` and `aglobalwith` with above
await this.useapi.awithuse(async () => {
await useapi.aglobalwith({
extension: mod.extension,
}, async () => {
// This is where the 'install' event gets triggered
await mod.install(mod_context);
});
});
}
}
}
};
_create_mod_context (parent, options) {
const path_ = require('path');
@ -380,6 +360,30 @@ class Kernel extends AdvancedBase {
const fs = require('fs');
const path_ = require('path');
// Expect main.js or index.js to exist
const options = ['main.js', 'index.js'];
// If no entry specified, find file with conventional name
if ( ! entry ) {
for ( const option of options ) {
if ( fs.existsSync(path_.join(mod_path, option)) ) {
entry = option;
break;
}
}
}
// If no entry specified or found, skip or error
if ( ! entry ) {
this.bootLogger.error(`Expected main.js or index.js in ${quot(mod_path)}`);
if ( ! process.env.SKIP_INVALID_MODS ) {
this.bootLogger.error(`Set SKIP_INVALID_MODS=1 (environment variable) to run anyway.`);
process.exit(1);
} else {
return;
}
}
const data = JSON.stringify({
name,
version: '1.0.0',

31
src/backend/src/PuterDriversModule.js
View File

@ -1,31 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
class PuterDriversModule extends AdvancedBase {
async install () {}
async install_legacy (context) {
const services = context.get('services');
const { DriverService } = require("./services/drivers/DriverService");
services.registerService('driver', DriverService);
}
}
module.exports = PuterDriversModule;

44
src/backend/src/api/APIError.js
View File

@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { URLSearchParams } = require("node:url");
const { quot } = require("../util/strutil");
const { quot } = require('@heyputer/putility').libs.string;
/**
* APIError represents an error that can be sent to the client.
@ -324,36 +324,6 @@ module.exports = class APIError {
message: () => 'Invalid token.',
},
// drivers
'interface_not_found': {
status: 404,
message: ({ interface_name }) => `Interface not found: ${quot(interface_name)}`,
},
'no_implementation_available': {
status: 502,
message: ({
iface,
interface_name,
driver
}) => `No implementation available for ` +
(iface ?? interface_name) ? 'interface' : 'driver' +
' ' + quot(iface ?? interface_name ?? driver) + '.',
},
'method_not_found': {
status: 404,
message: ({ interface_name, method_name }) => `Method not found: ${quot(method_name)} on interface ${quot(interface_name)}`,
},
'missing_required_argument': {
status: 400,
message: ({ interface_name, method_name, arg_name }) =>
`Missing required argument ${quot(arg_name)} for method ${quot(method_name)} on interface ${quot(interface_name)}`,
},
'argument_consolidation_failed': {
status: 400,
message: ({ interface_name, method_name, arg_name, message }) =>
`Failed to parse or process argument ${quot(arg_name)} for method ${quot(method_name)} on interface ${quot(interface_name)}: ${message}`,
},
// SLA
'rate_limit_exceeded': {
status: 429,
@ -505,18 +475,6 @@ module.exports = class APIError {
status: 400,
message: 'Incorrect or missing anti-CSRF token.',
},
// Chat
// TODO: specifying these errors here might be a violation
// of separation of concerns. Services could register their
// own errors with an error registry.
'max_tokens_exceeded': {
status: 400,
message: ({ input_tokens, max_tokens }) =>
`Input exceeds maximum token count. ` +
`Input has ${input_tokens} tokens, ` +
`but the maximum is ${max_tokens}.`,
},
};
/**

200
src/backend/src/api/eggspress.js
View File

@ -1,198 +1,2 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const express = require('express');
const multer = require('multer');
const multest = require('@heyputer/multest');
const api_error_handler = require('../api/api_error_handler.js');
const fsBeforeMW = require('../middleware/fs');
const APIError = require('./APIError.js');
const { Context } = require('../util/context.js');
/**
* eggspress() is a factory function for creating express routers.
*
* @param {*} route the route to the router
* @param {*} settings the settings for the router. The following
* properties are supported:
* - auth: whether or not to use the auth middleware
* - fs: whether or not to use the fs middleware
* - json: whether or not to use the json middleware
* - customArgs: custom arguments to pass to the router
* - allowedMethods: the allowed HTTP methods
* @param {*} handler the handler for the router
* @returns {express.Router} the router
*/
module.exports = function eggspress (route, settings, handler) {
const router = express.Router();
const mw = [];
const afterMW = [];
// These flags enable specific middleware.
if ( settings.abuse ) mw.push(require('../middleware/abuse')(settings.abuse));
if ( settings.auth ) mw.push(require('../middleware/auth'));
if ( settings.auth2 ) mw.push(require('../middleware/auth2'));
if ( settings.fs ) {
mw.push(fsBeforeMW);
}
if ( settings.verified ) mw.push(require('../middleware/verified'));
if ( settings.json ) mw.push(express.json());
// The `files` setting is an array of strings. Each string is the name
// of a multipart field that contains files. `multer` is used to parse
// the multipart request and store the files in `req.files`.
if ( settings.files ) {
for ( const key of settings.files ) {
mw.push(multer().array(key));
}
}
if ( settings.multest ) {
mw.push(multest());
}
// The `multipart_jsons` setting is an array of strings. Each string
// is the name of a multipart field that contains JSON. This middleware
// parses the JSON in each field and stores the result in `req.body`.
if ( settings.multipart_jsons ) {
for ( const key of settings.multipart_jsons ) {
mw.push((req, res, next) => {
try {
if ( ! Array.isArray(req.body[key]) ) {
req.body[key] = [JSON.parse(req.body[key])];
} else {
req.body[key] = req.body[key].map(JSON.parse);
}
} catch (e) {
return res.status(400).send({
error: {
message: `Invalid JSON in multipart field ${key}`
}
});
}
next();
});
}
}
// The `alias` setting is an object. Each key is the name of a
// parameter. Each value is the name of a parameter that should
// be aliased to the key.
if ( settings.alias ) {
for ( const alias in settings.alias ) {
const target = settings.alias[alias];
mw.push((req, res, next) => {
const values = req.method === 'GET' ? req.query : req.body;
if ( values[alias] ) {
values[target] = values[alias];
}
next();
});
}
}
// The `parameters` setting is an object. Each key is the name of a
// parameter. Each value is a `Param` object. The `Param` object
// specifies how to validate the parameter.
if ( settings.parameters ) {
for ( const key in settings.parameters ) {
const param = settings.parameters[key];
mw.push(async (req, res, next) => {
if ( ! req.values ) req.values = {};
const values = req.method === 'GET' ? req.query : req.body;
const getParam = (key) => values[key];
try {
const result = await param.consolidate({ req, getParam });
req.values[key] = result;
} catch (e) {
api_error_handler(e, req, res, next);
return;
}
next();
});
}
}
// what if I wanted to pass arguments to, for example, `json`?
if ( settings.customArgs ) mw.push(settings.customArgs);
if ( settings.alarm_timeout ) {
mw.push((req, res, next) => {
setTimeout(() => {
if ( ! res.headersSent ) {
const log = req.services.get('log-service').create('eggspress:timeout');
const errors = req.services.get('error-service').create(log);
let id = Array.isArray(route) ? route[0] : route;
id = id.replace(/\//g, '_');
errors.report(id, {
source: new Error('Response timed out.'),
message: 'Response timed out.',
trace: true,
alarm: true,
});
}
}, settings.alarm_timeout);
next();
});
}
if ( settings.response_timeout ) {
mw.push((req, res, next) => {
setTimeout(() => {
if ( ! res.headersSent ) {
api_error_handler(APIError.create('response_timeout'), req, res, next);
}
}, settings.response_timeout);
next();
});
}
if ( settings.mw ) mw.push(...settings.mw);
const errorHandledHandler = async function (req, res, next) {
if ( settings.subdomain ) {
if ( require('../helpers').subdomain(req) !== settings.subdomain ) {
return next();
}
}
try {
const expected_ctx = res.locals.ctx;
const received_ctx = Context.get(undefined, { allow_fallback: true });
if ( expected_ctx != received_ctx ) {
await expected_ctx.arun(async () => {
await handler(req, res, next);
});
} else await handler(req, res, next);
} catch (e) {
api_error_handler(e, req, res, next);
}
};
if ( settings.allowedMethods.includes('GET') ) {
router.get(route, ...mw, errorHandledHandler, ...afterMW);
}
if ( settings.allowedMethods.includes('POST') ) {
router.post(route, ...mw, errorHandledHandler, ...afterMW);
}
return router;
}
// This file is a legacy alias
module.exports = require('../modules/web/lib/eggspress.js');

3
src/backend/src/api/filesystem/UserParam.js
View File

@ -17,9 +17,6 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
module.exports = class UserParam {
constructor () {
//
}
consolidate ({ req }) {
return req.user;
}

27
src/backend/src/boot/RuntimeEnvironment.js
View File

@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
const { quot } = require("../util/strutil");
const { quot } = require('@heyputer/putility').libs.string;
const { TechnicalError } = require("../errors/TechnicalError");
const { print_error_help } = require("../errors/error_help_details");
const default_config = require("./default_config");
@ -93,8 +93,7 @@ const path_checks = ({ logger }) => ({ fs, path_ }) => ({
throw new Error(`No valid config file found in path: ${path}`);
},
env_not_set: name => () => {
if ( process.env[name] ) return false;
return true;
return ! process.env[name];
}
});
@ -233,18 +232,14 @@ class RuntimeEnvironment extends AdvancedBase {
]
);
// Note: there used to be a 'mods_path_entry' here too
// but it was never used
const pwd_path_entry = this.get_first_suitable_path_(
{ pathFor: 'working directory' },
this.runtime_paths,
[ this.path_checks.require_write_permission ]
);
const mods_path_entry = this.get_first_suitable_path_(
{ pathFor: 'mods', optional: true },
this.mod_paths,
[ this.path_checks.require_read_permission ],
);
process.chdir(pwd_path_entry.path);
// Check for a valid config file in the config path
@ -287,12 +282,11 @@ class RuntimeEnvironment extends AdvancedBase {
);
const config_values = JSON.parse(config_raw);
for ( const k in generated_values ) {
if ( config_values[k] ) {
if ( ! config_values[k] ) continue;
generated_values[k] = config_values[k];
}
}
}
}
const generated_config = {
...default_config,
...generated_values,
@ -334,9 +328,6 @@ class RuntimeEnvironment extends AdvancedBase {
throw new Error('config_name is required');
}
this.logger.info(hl(`config name`) + ` ${quot(config.config_name)}`);
// console.log(config.services);
// console.log(Object.keys(config.services));
// console.log({ ...config.services });
const mod_paths = [];
environment.mod_paths = mod_paths;
@ -363,12 +354,13 @@ class RuntimeEnvironment extends AdvancedBase {
}
get_first_suitable_path_ (meta, paths, last_checks) {
iter_paths:
for ( const entry of paths ) {
const checks = [...(entry.checks ?? []), ...last_checks];
this.logger.info(
`Checking path ${quot(entry.label ?? entry.path)} for ${meta.pathFor}...`
);
let checks_pass = true;
for ( const check of checks ) {
this.logger.info(
`-> doing ${quot(check.name)} on path ${quot(entry.path)}...`
@ -378,10 +370,13 @@ class RuntimeEnvironment extends AdvancedBase {
this.logger.info(
`-> ${quot(check.name)} doesn't like this path`
);
continue iter_paths;
checks_pass = false;
break;
}
}
if ( ! checks_pass ) continue;
this.logger.info(
`${hl('USING')} ${quot(entry.path)} for ${meta.pathFor}.`
)

25
src/backend/src/codex/CodeModel.js
View File

@ -1,25 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
class CodeModel {
static create () {}
}
module.exports = {
CodeModel,
};

11
src/backend/src/codex/Sequence.js
View File

@ -109,7 +109,7 @@ class Sequence {
async run (values) {
// Initialize scope
values = values || this.thisArg?.values || {};
this.scope_.__proto__ = values;
Object.setPrototypeOf(this.scope_, values);
// Run sequence
for ( ; this.i < this.steps.length ; this.i++ ) {
@ -126,9 +126,8 @@ class Sequence {
const parent_scope = this.scope_;
this.scope_ = {};
// We could do Object.assign(this.scope_, parent_scope), but
// setting __proto__ is faster because it leverages the optimizations
// of the JS engine for the prototype chain.
this.scope_.__proto__ = parent_scope;
// setting the prototype should be faster (in theory)
Object.setPrototypeOf(this.scope_, parent_scope);
if ( this.sequence_.options_.record_history ) {
this.value_history_.push(this.scope_);
@ -142,6 +141,10 @@ class Sequence {
this.thisArg, this,
);
if ( this.last_return_ instanceof Sequence.SequenceState ) {
this.scope_ = this.last_return_.scope_;
}
if ( this.sequence_.options_.after_each ) {
await this.sequence_.options_.after_each(this, step);
}

53
src/backend/src/config.js
View File

@ -31,24 +31,24 @@ config.disable_temp_users = false;
config.default_user_group = '78b1b1dd-c959-44d2-b02c-8735671f9997';
config.default_temp_group = 'b7220104-7905-4985-b996-649fdcdb3c8f';
config.max_file_size = 100_000_000_000,
config.max_thumb_size = 1_000,
config.max_fsentry_name_length = 767,
config.max_file_size = 100_000_000_000;
config.max_thumb_size = 1_000;
config.max_fsentry_name_length = 767;
config.username_regex = /^\w{1,}$/;
config.username_regex = /^\w+$/;
config.username_max_length = 45;
config.subdomain_regex = /^[a-zA-Z0-9-_-]+$/;
config.subdomain_regex = /^[a-zA-Z0-9_-]+$/;
config.subdomain_max_length = 60;
config.app_name_regex = /^[a-zA-Z0-9-_-]+$/;
config.app_name_regex = /^[a-zA-Z0-9_-]+$/;
config.app_name_max_length = 60;
config.app_title_max_length = 60;
config.min_pass_length = 6;
config.strict_email_verification_required = false,
config.require_email_verification_to_publish_website = false,
config.strict_email_verification_required = false;
config.require_email_verification_to_publish_website = false;
config.kv_max_key_size = 1024,
config.kv_max_value_size = 400 * 1024,
config.kv_max_key_size = 1024;
config.kv_max_value_size = 400 * 1024;
config.monitor = {
metricsInterval: 60000,
@ -70,9 +70,6 @@ config.app_max_icon_size = 5*1024*1024;
config.defaultjs_asset_path = '../../';
// config.origin = config.protocol + '://' + config.domain;
// config.api_base_url = config.protocol + '://api.' + config.domain;
// config.social_card = `${config.origin}/assets/img/screenshot.png`;
config.short_description = `Puter is a privacy-first personal cloud that houses all your files, apps, and games in one private and secure place, accessible from anywhere at any time.`;
config.title = 'Puter';
config.company = 'Puter Technologies Inc.';
@ -96,11 +93,12 @@ config.reserved_words = [];
// set default S3 settings for this server, if any
if (config.server_id) {
// see if this server has a specific bucket
for (let index = 0; index < config.servers.length; index++) {
if (config.servers[index].id === config.server_id && config.servers[index].s3_bucket){
config.s3_bucket = config.servers[index].s3_bucket;
config.s3_region = config.servers[index].region;
}
for ( const server of config.servers ) {
if ( server.id !== config.server_id ) continue;
if ( ! server.s3_bucket ) continue;
config.s3_bucket = server.s3_bucket;
config.s3_region = server.region;
}
}
@ -142,15 +140,15 @@ if ( config.os.refined ) {
module.exports = config;
// NEW_CONFIG_LOADING
const maybe_port = config =>
config.pub_port !== 80 && config.pub_port !== 443 ? ':' + config.pub_port : '';
const computed_defaults = {
pub_port: config => config.http_port,
origin: config => config.protocol + '://' + config.domain +
(config.pub_port !== 80 && config.pub_port !== 443 ? ':' + config.pub_port : ''),
origin: config => config.protocol + '://' + config.domain + maybe_port(config),
api_base_url: config => config.experimental_no_subdomain
? config.origin
: config.protocol + '://api.' + config.domain +
(config.pub_port !== 80 && config.pub_port !== 443 ? ':' + config.pub_port : ''),
: config.protocol + '://api.' + config.domain + maybe_port(config),
social_card: config => `${config.origin}/assets/img/screenshot.png`,
};
@ -162,7 +160,7 @@ let config_to_export;
// load_config() may replace
const config_pointer = {};
{
config_pointer.__proto__ = config;
Object.setPrototypeOf(config_pointer, config);
config_to_export = config_pointer;
}
@ -173,15 +171,14 @@ const config_pointer = {};
let replacement_config = {
...o,
};
// replacement_config.__proto__ = config_pointer.__proto__;
replacement_config = deep_proto_merge(replacement_config, config_pointer.__proto__, {
replacement_config = deep_proto_merge(replacement_config, Object.getPrototypeOf(config_pointer), {
preserve_flag: true,
})
config_pointer.__proto__ = replacement_config;
Object.setPrototypeOf(config_pointer, replacement_config);
};
const config_api = { load_config };
config_api.__proto__ = config_to_export;
Object.setPrototypeOf(config_api, config_to_export);
config_to_export = config_api;
}
@ -212,7 +209,7 @@ const config_pointer = {};
const config_runtime_values = {
$: 'runtime-values'
};
config_runtime_values.__proto__ = config_to_export;
Object.setPrototypeOf(config_runtime_values, config_to_export);
config_to_export = config_runtime_values
// These can be difficult to find and cause painful

2
src/backend/src/config/ConfigLoader.js
View File

@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
const { quot } = require("../util/strutil");
const { quot } = require('@heyputer/putility').libs.string;
class ConfigLoader extends AdvancedBase {
static MODULES = {

15
src/backend/src/data/hardcoded-permissions.js
View File

@ -84,17 +84,20 @@ const hardcoded_user_group_permissions = {
'service:hello-world:ii:hello-world': policy_perm('temp.es'),
'service:puter-kvstore:ii:puter-kvstore': policy_perm('temp.kv'),
'driver:puter-kvstore': policy_perm('temp.kv'),
'driver:puter-notifications': policy_perm('temp.es'),
'driver:puter-apps': policy_perm('temp.es'),
'driver:puter-subdomains': policy_perm('temp.es'),
'service:puter-notifications:ii:crud-q': policy_perm('temp.es'),
'service:puter-apps:ii:crud-q': policy_perm('temp.es'),
'service:puter-subdomains:ii:crud-q': policy_perm('temp.es'),
'service:es\\Cnotification:ii:crud-q': policy_perm('user.es'),
'service:es\\Capp:ii:crud-q': policy_perm('user.es'),
'service:es\\Csubdomain:ii:crud-q': policy_perm('user.es'),
},
'78b1b1dd-c959-44d2-b02c-8735671f9997': {
'service:hello-world:ii:hello-world': policy_perm('user.es'),
'service:puter-kvstore:ii:puter-kvstore': policy_perm('user.kv'),
'driver:puter-kvstore': policy_perm('user.kv'),
'driver:puter-notifications': policy_perm('user.es'),
'driver:puter-apps': policy_perm('user.es'),
'driver:puter-subdomains': policy_perm('user.es'),
'service:es\\Cnotification:ii:crud-q': policy_perm('user.es'),
'service:es\\Capp:ii:crud-q': policy_perm('user.es'),
'service:es\\Csubdomain:ii:crud-q': policy_perm('user.es'),
},
},
};

192
src/backend/src/definitions/Driver.js
View File

@ -1,192 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { AdvancedBase } = require("@heyputer/putility");
const { Context } = require('../util/context')
const APIError = require("../api/APIError");
const { AppUnderUserActorType, UserActorType } = require("../services/auth/Actor");
const { BaseOperation } = require("../services/OperationTraceService");
const { CodeUtil } = require("../codex/CodeUtil");
/**
* Base class for all driver implementations.
*
* @deprecated - we use traits on services now. This class is kept for compatibility
* with EntityStoreImplementation and DBKVStore which still use this.
*/
class Driver extends AdvancedBase {
constructor (...a) {
super(...a);
const methods = this._get_merged_static_object('METHODS');
// Turn each method into an operation
for ( const k in methods ) {
methods[k] = CodeUtil.mrwrap(methods[k], BaseOperation, {
name: `${this.constructor.ID}:${k}`,
});
};
this.methods = methods;
this.sla = this._get_merged_static_object('SLA');
}
async call (method, args) {
if ( ! this.methods[method] ) {
throw new Error(`method not found: ${method}`);
}
const pseudo_this = Object.assign({}, this);
const context = Context.get();
pseudo_this.context = context;
pseudo_this.services = context.get('services');
const services = context.get('services');
pseudo_this.log = services.get('log-service').create(this.constructor.name);
await this._sla_enforcement(method);
return await this.methods[method].call(pseudo_this, args);
}
async _sla_enforcement (method) {
const context = Context.get();
const services = context.get('services');
const method_key = `${this.constructor.ID}:${method}`;
const svc_sla = services.get('sla');
// System SLA enforcement
{
const sla_key = `driver:impl:${method_key}`;
const sla = await svc_sla.get('system', sla_key);
const sys_method_key = `system:${method_key}`;
// short-term rate limiting
if ( sla?.rate_limit ) {
const svc_rateLimit = services.get('rate-limit');
let eventual_success = false;
for ( let i = 0 ; i < 60 ; i++ ) {
try {
await svc_rateLimit.check_and_increment(sys_method_key, sla.rate_limit.max, sla.rate_limit.period);
eventual_success = true;
break;
} catch ( e ) {
if (
! ( e instanceof APIError ) ||
e.fields.code !== 'rate_limit_exceeded'
) throw e;
await new Promise((resolve) => setTimeout(resolve, 1000));
}
}
if ( ! eventual_success ) {
throw APIError.create('server_rate_exceeded');
}
}
}
// test_mode is checked to prevent rate limiting when it is enabled
const test_mode = context.get('test_mode');
// User SLA enforcement
{
const actor = context.get('actor').get_related_actor(UserActorType);
const user_is_verified = !! actor.type.user.email_confirmed;
const sla_key = `driver:impl:${method_key}`;
const sla = await svc_sla.get(
user_is_verified ? 'user_verified' : 'user_unverified',
sla_key
);
const user_method_key = `actor:${actor.uid}:${method_key}`;
// short-term rate limiting
if ( sla?.rate_limit ) {
const svc_rateLimit = services.get('rate-limit');
await svc_rateLimit.check_and_increment(method_key, sla.rate_limit.max, sla.rate_limit.period);
}
// long-term rate limiting
if ( sla?.monthly_limit && ! test_mode ) {
const svc_monthlyUsage = services.get('monthly-usage');
const count = await svc_monthlyUsage.check(
actor, {
'driver.interface': this.constructor.INTERFACE,
'driver.implementation': this.constructor.ID,
'driver.method': method,
});
if ( count >= sla.monthly_limit ) {
throw APIError.create('monthly_limit_exceeded', null, {
method_key,
limit: sla.monthly_limit,
});
}
}
}
// App SLA enforcement
await (async () => {
const actor = context.get('actor');
if ( ! ( actor.type instanceof AppUnderUserActorType ) ) return;
const sla_key = `driver:impl:${method_key}`;
const sla = await svc_sla.get('app_default', sla_key);
// long-term rate limiting
if ( sla?.monthly_limit && ! test_mode ) {
const svc_monthlyUsage = services.get('monthly-usage');
const count = await svc_monthlyUsage.check(
actor, {
'driver.interface': this.constructor.INTERFACE,
'driver.implementation': this.constructor.ID,
'driver.method': method,
});
if ( count >= sla.monthly_limit ) {
throw APIError.create('monthly_limit_exceeded', null, {
method_key,
limit: sla.monthly_limit,
});
}
}
})();
// Record monthly usage
if ( ! test_mode ) {
const actor = context.get('actor');
const svc_monthlyUsage = services.get('monthly-usage');
const extra = {
'driver.interface': this.constructor.INTERFACE,
'driver.implementation': this.constructor.ID,
'driver.method': method,
...(this.get_usage_extra ? this.get_usage_extra() : {}),
};
await svc_monthlyUsage.increment(actor, method_key, extra);
}
}
async get_response_meta () {
return {
driver: this.constructor.ID,
driver_version: this.constructor.VERSION,
driver_interface: this.constructor.INTERFACE,
};
}
}
module.exports = {
Driver,
};

217
src/backend/src/drivers/DBKVStore.js
View File

@ -1,217 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const config = require("../config");
const APIError = require("../api/APIError");
const { DB_READ, DB_WRITE } = require("../services/database/consts");
const { Driver } = require("../definitions/Driver");
const { get_app } = require("../helpers");
class DBKVStore extends Driver {
static ID = 'public-db-kvstore';
static VERSION = '0.0.0';
static INTERFACE = 'puter-kvstore';
static MODULES = {
murmurhash: require('murmurhash'),
}
static METHODS = {
get: async function ({ app_uid, key }) {
const actor = this.context.get('actor');
// If the actor is an app then it gets its own KV store.
// The way this is implemented isn't ideal for future behaviour;
// a KV implementation specified by the user would have parameters
// that are scoped to the app, so this should eventually be
// changed to get the app ID from the same interface that would
// be used to obtain per-app user-specified implementation params.
let app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( ! user ) throw new Error('User not found');
if ( ! app && app_uid ) {
app = await get_app({ uid: app_uid });
}
const db = this.services.get('database').get(DB_READ, 'kvstore');
const key_hash = this.modules.murmurhash.v3(key);
const kv = app ? await db.read(
`SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1`,
[ user.id, app.uid, key_hash ]
) : await db.read(
`SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global') AND kkey_hash=? LIMIT 1`,
[ user.id, key_hash ]
);
if ( kv[0] ) kv[0].value = db.case({
mysql: () => kv[0].value,
otherwise: () => JSON.parse(kv[0].value ?? 'null'),
})();
return kv[0]?.value ?? null;
},
set: async function ({ app_uid, key, value }) {
const actor = this.context.get('actor');
// Validate the key
// get() doesn't String() the key but it only passes it to
// murmurhash.v3() so it doesn't need to ¯\_(ツ)_/¯
key = String(key);
if ( Buffer.byteLength(key, 'utf8') > config.kv_max_key_size ) {
throw new Error(`key is too large. Max size is ${config.kv_max_key_size}.`);
}
// Validate the value
value = value === undefined ? null : value;
if (
value !== null &&
Buffer.byteLength(JSON.stringify(value), 'utf8') >
config.kv_max_value_size
) {
throw new Error(`value is too large. Max size is ${config.kv_max_value_size}.`);
}
let app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( ! user ) throw new Error('User not found');
if ( ! app && app_uid ) {
app = await get_app({ uid: app_uid });
}
const db = this.services.get('database').get(DB_WRITE, 'kvstore');
const key_hash = this.modules.murmurhash.v3(key);
try {
await db.write(
`INSERT INTO kv (user_id, app, kkey_hash, kkey, value)
VALUES (?, ?, ?, ?, ?) ` +
db.case({
mysql: 'ON DUPLICATE KEY UPDATE value = ?',
sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET value = excluded.value',
}),
[
user.id, app?.uid ?? 'global', key_hash, key,
JSON.stringify(value),
...db.case({ mysql: [value], otherwise: [] }),
]
);
} catch (e) {
// I discovered that my .sqlite file was corrupted and the update
// above didn't work. The current database initialization does not
// cause this issue so I'm adding this log as a safeguard.
// - KernelDeimos / ED
const svc_error = this.services.get('error-service');
svc_error.report('kvstore:sqlite_error', {
message: 'Broken database version - please contact maintainers',
source: e,
});
}
return true;
},
del: async function ({ app_uid, key }) {
const actor = this.context.get('actor');
let app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( ! user ) throw new Error('User not found');
if ( ! app && app_uid ) {
app = await get_app({ uid: app_uid });
}
const db = this.services.get('database').get(DB_WRITE, 'kvstore');
const key_hash = this.modules.murmurhash.v3(key);
await db.write(
`DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?`,
[ user.id, app?.uid ?? 'global', key_hash ]
);
return true;
},
list: async function ({ app_uid, as }) {
const actor = this.context.get('actor');
let app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( ! app && app_uid ) {
app = await get_app({ uid: app_uid });
}
if ( ! user ) throw new Error('User not found');
const db = this.services.get('database').get(DB_READ, 'kvstore');
let rows = app ? await db.read(
`SELECT kkey, value FROM kv WHERE user_id=? AND app=?`,
[ user.id, app.uid ]
) : await db.read(
`SELECT kkey, value FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global')`,
[ user.id ]
);
rows = rows.map(row => ({
key: row.kkey,
value: db.case({
mysql: () => row.value,
otherwise: () => JSON.parse(row.value ?? 'null')
})(),
}));
as = as || 'entries';
if ( ! ['keys','values','entries'].includes(as) ) {
throw APIError.create('field_invalid', null, {
key: 'as',
expected: '"keys", "values", or "entries"',
});
}
if ( as === 'keys' ) rows = rows.map(row => row.key);
else if ( as === 'values' ) rows = rows.map(row => row.value);
return rows;
},
flush: async function ({ app_uid }) {
const actor = this.context.get('actor');
let app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( ! user ) throw new Error('User not found');
if ( ! app && app_uid ) {
app = await get_app({ uid: app_uid });
}
const db = this.services.get('database').get(DB_WRITE, 'kvstore');
await db.write(
`DELETE FROM kv WHERE user_id=? AND app=?`,
[ user.id, app?.uid ?? 'global' ]
);
return true;
}
}
}
module.exports = {
DBKVStore,
}

171
src/backend/src/drivers/EntityStoreImplementation.js
View File

@ -1,171 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const APIError = require("../api/APIError");
const { Driver } = require("../definitions/Driver");
const { Entity } = require("../om/entitystorage/Entity");
const { Or, And, Eq } = require("../om/query/query");
const _fetch_based_on_complex_id = async (self, id) => {
// Ensure `id` is an object and get its keys
if ( ! id || typeof id !== 'object' || Array.isArray(id) ) {
throw APIError.create('invalid_id', null, { id });
}
const id_keys = Object.keys(id);
// sort keys alphabetically
id_keys.sort();
// Ensure key set is valid based on redundant keys listing
const svc_es = self.services.get(self.service);
const redundant_identifiers = svc_es.om.redundant_identifiers ?? [];
let match_found = false;
for ( let key of redundant_identifiers ) {
// Either a single key or a list
key = Array.isArray(key) ? key : [key];
// All keys in the list must be present in the id
for ( let i=0 ; i < key.length ; i++ ) {
if ( ! id_keys.includes(key[i]) ) {
break;
}
if ( i === key.length - 1 ) {
match_found = true;
break;
}
}
}
if ( ! match_found ) {
throw APIError.create('invalid_id', null, { id });
}
// Construct a query predicate based on the keys
const key_eqs = [];
for ( const key of id_keys ) {
key_eqs.push(new Eq({
key,
value: id[key],
}));
}
let predicate = new And({ children: key_eqs });
// Perform a select
const entity = await svc_es.read({ predicate });
if ( ! entity ) {
return null;
}
// Ensure there is only one result
return entity;
}
const _fetch_based_on_either_id = async (self, uid, id) => {
if ( uid ) {
const svc_es = self.services.get(self.service);
return await svc_es.read(uid);
}
return await _fetch_based_on_complex_id(self, id);
}
class EntityStoreImplementation extends Driver {
constructor ({ service }) {
super();
this.service = service;
}
get_usage_extra () {
return {
['driver.interface']: 'puter-es',
['driver.implementation']: 'puter-es:' + this.service,
};
}
static METHODS = {
create: async function ({ object, options }) {
const svc_es = this.services.get(this.service);
if ( object.hasOwnProperty(svc_es.om.primary_identifier) ) {
throw APIError.create('field_not_allowed_for_create', null, { key: svc_es.om.primary_identifier });
}
const entity = await Entity.create({ om: svc_es.om }, object);
return await svc_es.create(entity, options);
},
update: async function ({ object, id, options }) {
const svc_es = this.services.get(this.service);
// if ( ! object.hasOwnProperty(svc_es.om.primary_identifier) ) {
// throw APIError.create('field_required_for_update', null, { key: svc_es.om.primary_identifier });
// }
const entity = await Entity.create({ om: svc_es.om }, object);
return await svc_es.update(entity, id, options);
},
upsert: async function ({ object, id, options }) {
const svc_es = this.services.get(this.service);
const entity = await Entity.create({ om: svc_es.om }, object);
return await svc_es.upsert(entity, id, options);
},
read: async function ({ uid, id }) {
if ( ! uid && ! id ) {
throw APIError.create('xor_field_missing', null, {
names: ['uid', 'id'],
});
}
const entity = await _fetch_based_on_either_id(this, uid, id);
if ( ! entity ) {
throw APIError.create('entity_not_found', null, {
identifier: uid
});
}
return await entity.get_client_safe();
},
select: async function (options) {
const svc_es = this.services.get(this.service);
const entities = await svc_es.select(options);
const client_safe_entities = [];
for ( const entity of entities ) {
client_safe_entities.push(await entity.get_client_safe());
}
return client_safe_entities;
},
delete: async function ({ uid, id }) {
if ( ! uid && ! id ) {
throw APIError.create('xor_field_missing', null, {
names: ['uid', 'id'],
});
}
if ( id && ! uid ) {
const entity = await _fetch_based_on_complex_id(this, id);
if ( ! entity ) {
throw APIError.create('entity_not_found', null, {
identifier: id
});
}
const svc_es = this.services.get(this.service);
uid = await entity.get(svc_es.om.primary_identifier);
}
const svc_es = this.services.get(this.service);
return await svc_es.delete(uid);
},
};
}
module.exports = {
EntityStoreImplementation,
};

43
src/backend/src/drivers/HelloWorld.js
View File

@ -1,43 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { Driver } = require("../definitions/Driver");
class HelloWorld extends Driver {
static ID = 'public-helloworld';
static VERSION = '0.0.0';
static INTERFACE = 'helloworld';
static SLA = {
greet: {
rate_limit: {
max: 10,
period: 30000,
},
monthly_limit: Math.pow(1, 6),
},
}
static METHODS = {
greet: async function ({ subject }) {
return `Hello, ${subject ?? 'World'}!`
}
}
}
module.exports = {
HelloWorld,
};

16
src/backend/src/errors/error_help_details.js
View File

@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { quot, osclink } = require("../util/strutil");
const { quot, osclink } = require('@heyputer/putility').libs.string;
const reused = {
runtime_env_references: [
@ -50,7 +50,7 @@ const error_help_details = [
apply (more) {
more.references = [
...reused.runtime_env_references,
]
];
}
},
{
@ -68,10 +68,10 @@ const error_help_details = [
{
title: 'Set CONFIG_PATH or RUNTIME_PATH environment variable',
},
],
];
more.references = [
...reused.runtime_env_references,
]
];
}
},
{
@ -83,7 +83,7 @@ const error_help_details = [
{
title: 'Create a valid config file',
},
]
];
}
},
{
@ -112,7 +112,7 @@ const error_help_details = [
use: 'describes why this error occurs',
url: 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Invalid_const_assignment'
},
]
];
}
},
{
@ -122,7 +122,7 @@ const error_help_details = [
apply (more) {
more.notes = [
'It looks like this might be our fault.',
]
];
more.solutions = [
{
title: `Check for an issue on ` +
@ -135,7 +135,7 @@ const error_help_details = [
'create one'
) + '.'
}
]
];
}
},
{

34
src/backend/src/filesystem/FSNodeContext.js
View File

@ -18,11 +18,13 @@
*/
const { get_user, get_dir_size, id2path, id2uuid, is_empty, is_shared_with_anyone, suggest_app_for_fsentry, get_app } = require("../helpers");
const putility = require('@heyputer/putility');
const { MultiDetachable } = putility.libs.listener;
const { TDetachable } = putility.traits;
const config = require("../config");
const _path = require('path');
const { NodeInternalIDSelector, NodeChildSelector, NodeUIDSelector, RootNodeSelector, NodePathSelector } = require("./node/selectors");
const { Context } = require("../util/context");
const { MultiDetachable } = require("../util/listenerutil");
const { NodeRawEntrySelector } = require("./node/selectors");
const { DB_READ } = require("../services/database/consts");
const { UserActorType } = require("../services/auth/Actor");
@ -87,7 +89,7 @@ module.exports = class FSNodeContext {
this.fs = fs;
// Decorate all fetch methods with otel span
// TODO: language tool for traits; this is a trait
// TODO: Apply method decorators using a putility class feature
const fetch_methods = [
'fetchEntry',
'fetchPath',
@ -271,8 +273,6 @@ module.exports = class FSNodeContext {
resourceService,
} = Context.get('services').values;
// await this.fs.resourceService
// .waitForResource(this.selector);
if ( fetch_entry_options.tracer == null ) {
fetch_entry_options.tracer = traceService.tracer;
}
@ -288,12 +288,8 @@ module.exports = class FSNodeContext {
await new Promise (rslv => {
const detachables = new MultiDetachable();
let resolved = false;
const callback = (resolver) => {
// NOTE: commented out for now because it's too verbose
resolved = true;
detachables.detach();
detachables.as(TDetachable).detach();
rslv();
}
@ -499,8 +495,7 @@ module.exports = class FSNodeContext {
[this.entry.id]
);
const versions_tidy = [];
for (let index = 0; index < versions.length; index++) {
const version = versions[index];
for ( const version of versions ) {
let username = version.user_id ? (await get_user({id: version.user_id})).username : null;
versions_tidy.push({
id: version.version_id,
@ -518,13 +513,15 @@ module.exports = class FSNodeContext {
/**
* Fetches the size of a file or directory if it was not
* already fetched.
* @param {object} user the user is needed to fetch the size
*/
async fetchSize (user) {
async fetchSize () {
const { fsEntryService } = Context.get('services').values;
// we already have the size for files
if ( ! this.entry.is_dir ) return;
if ( ! this.entry.is_dir ) {
await this.fetchEntry();
return this.entry.size;
}
this.entry.size = await fsEntryService.get_recursive_size(
this.entry.uuid,
@ -551,14 +548,6 @@ module.exports = class FSNodeContext {
this.entry.is_empty = await is_empty(this.uid);
}
// TODO: this is currently not called anywhere; for now it
// will never be fetched since sharing is not a priority.
async fetchIsShared () {
if ( ! this.mysql_id ) return;
this.entry.is_shared = await is_shared_with_anyone(this.mysql_id);
}
async fetchAll(fsEntryFetcher, user, force) {
await this.fetchEntry({ thumbnail: true });
await this.fetchSubdomains(user);
@ -610,7 +599,6 @@ module.exports = class FSNodeContext {
);
}
if ( ! this.path ) {
// console.log('PATH WAS NOT ON ENTRY', this);
await this.fetchPath();
}
if ( ! this.path ) {

324
src/backend/src/filesystem/FSOperationContext.js
View File

@ -1,324 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const PerformanceMonitor = require('../monitor/PerformanceMonitor');
const FSNodeContext = require('./FSNodeContext');
const FSAccessContext = require('./FSAccessContext');
const { Context } = require('../util/context');
/**
* FSOperationContext represents a single operation on the filesystem.
*
* FSOperationContext is used to record events such as side-effects
* which occur during a high-level filesystem operation. It is also
* responsible for generating a client-safe result which describes
* the operation.
*/
module.exports = class FSOperationContext {
// TODO: rename this.fs to this.access
constructor (op_name, context, options) {
// migration: fs:create-service
// TODO: rename this.fs to this.access
// NOTE: the 2nd parameter of this constructor
// was called `fs` and was expected to be FSAccessContext.
// Now it should be a context object holding the services
// container. context.access is the FSAccessContext.
if ( context instanceof FSAccessContext ) {
this.fs = context;
} else if ( context ) {
this.context = context;
this.fs = context.access;
} else {
const x = Context.get();
this.fs = {};
this.fs.traceService = x.get('services').get('traceService');
}
this.name = op_name;
this.events = [];
this.parent_dirs_created = [];
this.created = [];
this.fields = {};
this.safeFields = {};
this.valueListeners_ = {};
this.valueFactories_ = {};
this.values_ = {};
this.rejections_ = {};
this.tasks_ = [];
this.currentCheckpoint_ = 'checkpoint not set';
if ( options.parent_operation ) {
this.parent = options.parent_operation;
}
this.donePromise = new Promise((resolve, reject) => {
this.doneResolve = resolve;
this.doneReject = reject;
});
// migration: arch:trace-service:move-outta-fs
if ( this.fs.traceService ) {
// Set 'span_' to current active span
const { context, trace } = require('@opentelemetry/api');
this.span_ = trace.getSpan(context.active());
}
this.monitor = PerformanceMonitor.createContext(`fs.${op_name}`);
}
checkpoint (label) {
this.currentCheckpoint_ = label;
}
async addTask (name, fn) {
const task = {
name,
operations: [],
promise: Promise.resolve(),
};
const taskContext = {
registerOperation: op => {
task.operations.push(op);
task.promise = task.promise.then(() => op.awaitDone());
}
};
const monitor = PerformanceMonitor.createContext('fs.rm');
monitor.label(`task:${name}`);
task.promise = task.promise.then(() => fn(taskContext));
this.tasks_.push(task);
let last_promise = null;
while ( task.promise !== last_promise ) {
last_promise = task.promise;
await task.promise;
}
// await task.promise;
monitor.stamp();
monitor.end();
}
get span () { return this.span_; }
recordParentDirCreated (fsNode) {
if ( ! fsNode ) {
throw new Error(
'falsy value to recordParentDirCreated',
fsNode,
);
}
this.parent_dirs_created.push(fsNode);
}
recordCreated (fsNode) {
this.created.push(fsNode);
}
set (field, value) {
this.fields[field] = value;
}
async set_now (field, value) {
this.fields[field] = value;
if ( value instanceof FSNodeContext ) {
this.safeFields[field] = await value.getSafeEntry();
}
}
get (field) {
return this.fields[field];
}
complete (options) {
options = options ?? {};
if ( this.parent ) {
for ( const fsNode of this.parent_dirs_created ) {
this.parent.recordParentDirCreated(fsNode);
}
for ( const fsNode of this.created ) {
this.parent.recordCreated(fsNode);
}
}
if ( this.tasks_.length > 0 ) {
// TODO: it's mutating input options, which is not ideal
if ( ! options.after ) options.after = [];
options.after.push(
this.tasks_.map(task => task.promise)
);
}
if ( options.after ) {
const thingsToWaitFor = options.after.map(item => {
if ( item.awaitDone ) return item.awaitDone;
return item;
});
(async () => {
await Promise.all(thingsToWaitFor);
this.doneResolve();
})();
return;
}
this.doneResolve();
}
onComplete(fn) {
this.donePromise.then(fn);
}
awaitDone () {
return this.donePromise;
}
provideValue (key, value) {
this.values_[key] = value;
let listeners = this.valueListeners_[key];
if ( ! listeners ) return;
delete this.valueListeners_[key];
for ( let listener of listeners ) {
if ( Array.isArray(listener) ) listener = listener[0];
listener(value);
}
}
rejectValue (key, err) {
this.rejections_[key] = err;
let listeners = this.valueListeners_[key];
if ( ! listeners ) return;
delete this.valueListeners_[key];
for ( let listener of listeners ) {
if ( ! Array.isArray(listener) ) continue;
if ( ! listener[1] ) continue;
listener = listener[1];
listener(err);
}
}
awaitValue (key) {
return new Promise ((rslv, rjct) => {
this.onValue(key, rslv, rjct);
});
}
onValue (key, fn, rjct) {
if ( this.values_[key] ) {
fn(this.values_[key]);
return;
}
if ( this.rejections_[key] ) {
if ( rjct ) {
rjct(this.rejections_[key]);
} else throw this.rejections_[key];
return;
}
if ( ! this.valueListeners_[key] ) {
this.valueListeners_[key] = [];
}
this.valueListeners_[key].push([fn, rjct]);
if ( this.valueFactories_[key] ) {
const fn = this.valueFactories_[key];
delete this.valueFactories_[key];
(async () => {
try {
const value = await fn();
this.provideValue(key, value);
} catch (e) {
this.rejectValue(key, e);
}
})();
}
}
async setFactory (key, factoryFn) {
if ( this.valueListeners_[key] ) {
let v;
try {
v = await factoryFn();
} catch (e) {
this.rejectValue(key, e);
}
this.provideValue(key, v);
return;
}
this.valueFactories_[key] = factoryFn;
}
/**
* Listen for another operation to complete, and then
* complete this operation. This is useful for operations
* which delegate to other operations.
*
* @param {FSOperationContext} other
* @returns {FSOperationContext} this
*/
completedBy (other) {
other.onComplete(() => {
this.complete();
});
return this;
}
/**
* Produces an object which describes the operation in a
* way that is intended to be sent to the client.
*
* @returns {Promise<Object>}
*/
async getClientSafeResult () {
const result = {};
for ( const field in this.fields ) {
if ( this.fields[field] instanceof FSNodeContext ) {
result[field] = this.safeFields[field] ??
await this.fields[field].getSafeEntry();
continue;
}
result[field] = this.fields[field];
}
result.parent_dirs_created = [];
for ( const fsNode of this.parent_dirs_created ) {
const fsNodeResult = await fsNode.getSafeEntry();
result.parent_dirs_created.push(fsNodeResult);
}
return result;
}
}

50
src/backend/src/filesystem/FilesystemService.js
View File

@ -17,13 +17,9 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// TODO: database access can be a service
const { ResourceService, RESOURCE_STATUS_PENDING_CREATE } = require('./storage/ResourceService');
const DatabaseFSEntryFetcher = require("./storage/DatabaseFSEntryFetcher");
const { DatabaseFSEntryService } = require('./storage/DatabaseFSEntryService');
const { SizeService } = require('./storage/SizeService');
const { RESOURCE_STATUS_PENDING_CREATE } = require('../modules/puterfs/ResourceService.js');
const { TraceService } = require('../services/TraceService.js');
const FSAccessContext = require('./FSAccessContext.js');
const SystemFSEntryService = require('./storage/SystemFSEntryService.js');
const PerformanceMonitor = require('../monitor/PerformanceMonitor.js');
const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector } = require('./node/selectors.js');
const FSNodeContext = require('./FSNodeContext.js');
@ -44,29 +40,15 @@ class FilesystemService extends BaseService {
static MODULES = {
_path: require('path'),
uuidv4: require('uuid').v4,
socketio: require('../socketio.js'),
config: require('../config.js'),
}
old_constructor (args) {
// super(args);
const { services } = args;
// this.services = services;
services.registerService('resourceService', ResourceService);
services.registerService('sizeService', SizeService);
services.registerService('traceService', TraceService);
// TODO: [fs:remove-separate-updater-and-fetcher]
services.set('fsEntryFetcher', new DatabaseFSEntryFetcher({
services: services,
}));
services.registerService('fsEntryService', DatabaseFSEntryService);
// The new fs entry service
services.registerService('systemFSEntryService', SystemFSEntryService);
this.log = services.get('log-service').create('filesystem-service');
// used by update_child_paths
@ -81,27 +63,6 @@ class FilesystemService extends BaseService {
.obtain('fs.fsentry:path')
.exec(entry.uuid);
});
// Decorate methods with otel span
// TODO: language tool for traits; this is a trait
const span_methods = [
'write', 'mkdir', 'rm', 'mv', 'cp', 'read', 'stat',
'mkdir_2',
'update_child_paths',
];
for ( const method of span_methods ) {
const original_method = this[method];
this[method] = async (...args) => {
const tracer = services.get('traceService').tracer;
let result;
await tracer.startActiveSpan(`fs-svc:${method}`, async span => {
result = await original_method.call(this, ...args);
span.end();
});
return result;
}
}
}
async _init () {
@ -261,8 +222,7 @@ class FilesystemService extends BaseService {
await target.fetchEntry({ thumbnail: true });
const { _path, uuidv4 } = this.modules;
const resourceService = this.services.get('resourceService');
const systemFSEntryService = this.services.get('systemFSEntryService');
const svc_fsEntry = this.services.get('fsEntryService');
const ts = Math.round(Date.now() / 1000);
const uid = uuidv4();
@ -292,7 +252,7 @@ class FilesystemService extends BaseService {
this.log.debug('creating fsentry', { fsentry: raw_fsentry })
const entryOp = await systemFSEntryService.insert(raw_fsentry);
const entryOp = await svc_fsEntry.insert(raw_fsentry);
console.log('entry op', entryOp);
@ -329,7 +289,7 @@ class FilesystemService extends BaseService {
const { _path, uuidv4 } = this.modules;
const resourceService = this.services.get('resourceService');
const systemFSEntryService = this.services.get('systemFSEntryService');
const svc_fsEntry = this.services.get('fsEntryService');
const ts = Math.round(Date.now() / 1000);
const uid = uuidv4();
@ -356,7 +316,7 @@ class FilesystemService extends BaseService {
this.log.debug('creating symlink', { fsentry: raw_fsentry })
const entryOp = await systemFSEntryService.insert(raw_fsentry);
const entryOp = await svc_fsEntry.insert(raw_fsentry);
(async () => {
await entryOp.awaitDone();

24
src/backend/src/filesystem/backends/Test.js
View File

@ -1,24 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Test = void 0;
class Test {
}
exports.Test = Test;

3
src/backend/src/filesystem/backends/Test.ts
View File

@ -1,3 +0,0 @@
export class Test {
//
}

4
src/backend/src/filesystem/batch/BatchExecutor.js
View File

@ -19,11 +19,11 @@
const { AdvancedBase } = require('@heyputer/putility');
const PathResolver = require('../../routers/filesystem_api/batch/PathResolver');
const commands = require('./commands').commands;
const { WorkUnit } = require('../../services/runtime-analysis/ExpectationService');
const APIError = require('../../api/APIError');
const { Context } = require('../../util/context');
const config = require('../../config');
const { TeePromise } = require('../../util/promise');
const { TeePromise } = require('@heyputer/putility').libs.promise;
const { WorkUnit } = require('../../modules/core/lib/expect');
class BatchExecutor extends AdvancedBase {
constructor (x, { actor, log, errors }) {

2
src/backend/src/filesystem/core/.gitignore vendored
View File

@ -1,2 +0,0 @@
# Typescript directory
*.js

85
src/backend/src/filesystem/core/BackendAPI.ts
View File

@ -1,85 +0,0 @@
import { ISelector } from "./Selector";
type PuterUserID = number;
export const enum FSBackendSupportFlags {
None = 0,
// Platform-related flags
PlatformCaseSensitive = 1 << 1,
// Puter support flags
// PuterStatOwner indicates the backend can store `user_id`
PuterStatOwner = 1 << 2,
// PuterStatApp indicates the backend can store `associated_app_id`
PuterStatApp = 1 << 3,
// DetailVerboseReaddir indicates the backend will provide a full
// stat() result for each entry in readdir().
DetailVerboseReaddir = 1 << 4,
}
export const enum FSNodeType {
File,
Directory,
PuterShortcut,
SymbolicLink,
KVStore,
Socket,
}
export interface IOverwriteOptions {
readonly overwrite: boolean;
UserID: PuterUserID,
}
export interface IWriteOptions extends IOverwriteOptions {
readonly create: boolean;
}
export interface IDeleteOptions {
readonly recursive: boolean;
}
export interface IStatOptions {
followSymlinks?: boolean;
}
export interface IStatResult {
uuid: string;
name: string;
type: FSNodeType;
size: number;
mtime: Date;
ctime: Date;
atime: Date;
immutable: boolean;
}
export interface IMiniStatResult {
uuid: string;
name: string;
type: FSNodeType;
}
type ReaddirResult = IMiniStatResult | IStatResult;
export interface IMkdirOptions {
// Not for permission checks by the storage backend.
// A supporting storage backend will simply store this and
// return it in the stat() call.
UserID: PuterUserID,
}
export interface BackendAPI {
stat (selector: ISelector, options: IStatOptions): Promise<IStatResult>;
readdir (selector: ISelector): Promise<[string, ReaddirResult][]>;
mkdir (selector: ISelector, name: string): Promise<void>;
copy (from: ISelector, to: ISelector, options: IOverwriteOptions): Promise<void>;
rename (from: ISelector, to: ISelector, options: IOverwriteOptions): Promise<void>;
delete (selector: ISelector, options: IDeleteOptions): Promise<void>;
read_file (selector: ISelector): Promise<Buffer>;
write_file (selector: ISelector, data: Buffer, options: IOverwriteOptions): Promise<void>;
}

0
src/backend/src/filesystem/core/FSEntry.ts
View File

65
src/backend/src/filesystem/core/Selector.ts
View File

@ -1,65 +0,0 @@
import * as _path from 'path';
import * as _util from 'util';
type TemporeryNodeType = any;
export interface ISelector {
describe (showDebug?: boolean): string;
setPropertiesKnownBySelector (node: object): void;
}
export class NodePathSelector {
public value: string;
constructor (path: string) {
this.value = path;
}
public describe (showDebug?: boolean): string {
return this.value;
}
public setPropertiesKnownBySelector (node: TemporeryNodeType): void {
node.path = this.value;
node.name = _path.basename(this.value);
}
}
export class NodeInternalUIDSelector {
public value: string;
constructor (uid: string) {
this.value = uid;
}
public describe (showDebug?: boolean): string {
return `[uid:${this.value}]`;
}
public setPropertiesKnownBySelector (node: TemporeryNodeType): void {
node.uid = this.value;
}
}
export class NodeInternalIDSelector {
constructor (
public service: string,
public id: number,
public debugInfo: any
) { }
public describe (showDebug?: boolean): string {
if ( showDebug ) {
return `[db:${this.id}] (${
_util.inspect(this.debugInfo)
})`;
}
return `[db:${this.id}]`;
}
public setPropertiesKnownBySelector (node: TemporeryNodeType): void {
if ( this.service === 'mysql' ) {
node.id = this.id;
}
}
}

4
src/backend/src/filesystem/hl_operations/hl_copy.js
View File

@ -141,8 +141,8 @@ class HLCopy extends HLFilesystemOperation {
const sizeService = svc.get('sizeService');
let deset_usage = await sizeService.get_usage(dest_user.id);
const size = await source.fetchSize(values.user);
let capacity = config.is_storage_limited ? (dest_user.free_storage === undefined || dest_user.free_storage === null) ? config.storage_capacity : dest_user.free_storage : config.available_device_storage
const size = await source.fetchSize();
const capacity = await sizeService.get_storage_capacity(dest_user.id);
if(capacity - deset_usage - size < 0){
throw APIError.create('storage_limit_reached');
}

3
src/backend/src/filesystem/hl_operations/hl_data_read.js
View File

@ -75,14 +75,11 @@ class HLDataRead extends HLFilesystemOperation {
const output_stream = new PassThrough();
new Promise((resolve, reject) => {
rl.on('line', (line) => {
output_stream.write(line);
});
rl.on('close', () => {
output_stream.end();
resolve();
});
});
return output_stream;

13
src/backend/src/filesystem/hl_operations/hl_mkdir.js
View File

@ -73,8 +73,7 @@ class MkTree extends HLFilesystemOperation {
}
async create_branch_ ({ parent_node, tree, parent_exists }) {
const { context, values } = this;
const { _path } = this.modules;
const { context } = this;
const fs = context.get('services').get('filesystem');
const actor = context.get('actor');
@ -82,7 +81,6 @@ class MkTree extends HLFilesystemOperation {
const branches = tree.slice(1);
let current = parent_node.selector;
let lastCreatedSelector = parent_node.selector;
// trunk = a/b/c
@ -242,7 +240,6 @@ class HLMkdir extends HLFilesystemOperation {
static MODULES = {
_path: require('path'),
socketio: require('../../socketio.js'),
}
static PROPERTIES = {
@ -258,7 +255,7 @@ class HLMkdir extends HLFilesystemOperation {
async _run () {
const { context, values } = this;
const { _path, socketio } = this.modules;
const { _path } = this.modules;
const fs = context.get('services').get('filesystem');
if ( ! is_valid_path(values.path, {
@ -385,12 +382,8 @@ class HLMkdir extends HLFilesystemOperation {
}
async _create_parents ({ parent_node }) {
const { context, values } = this;
const { values } = this;
const { _path } = this.modules;
const fs = context.get('services').get('filesystem');
let current = parent_node.selector;
let lastCreatedSelector = null;
const tree_op = new MkTree();
await tree_op.run({

3
src/backend/src/filesystem/hl_operations/hl_mklink.js
View File

@ -36,11 +36,10 @@ class HLMkLink extends HLFilesystemOperation {
async _run () {
const { context, values } = this;
const { _path } = this.modules;
const fs = context.get('services').get('filesystem');
const { target, parent, user } = values;
let { name, dedupe_name } = values;
let { name } = values;
if ( ! name ) {
throw APIError.create('field_empty', null, { key: 'name' });

1
src/backend/src/filesystem/hl_operations/hl_mkshortcut.js
View File

@ -40,7 +40,6 @@ class HLMkShortcut extends HLFilesystemOperation {
async _run () {
console.log('HLMKSHORTCUT IS HAPPENING')
const { context, values } = this;
const { _path, socketio } = this.modules;
const fs = context.get('services').get('filesystem');
const { target, parent, user } = values;

3
src/backend/src/filesystem/hl_operations/hl_move.js
View File

@ -106,7 +106,8 @@ class HLMove extends HLFilesystemOperation {
dest_user = source_user;
await source.fetchSize();
const item_size = source.entry.size;
let capacity = config.is_storage_limited ? (dest_user.free_storage === undefined || dest_user.free_storage === null) ? config.storage_capacity : dest_user.free_storage : config.available_device_storage;
const sizeService = svc.get('sizeService');
const capacity = await sizeService.get_storage_capacity(user.id);
if(capacity - await df(dest_user.id) - item_size < 0){
throw APIError.create('storage_limit_reached');
}

1
src/backend/src/filesystem/hl_operations/hl_read.js
View File

@ -28,7 +28,6 @@ class HLRead extends HLFilesystemOperation {
}
async _run () {
const { context } = this;
const {
fsNode, actor,
line_count, byte_count,

7
src/backend/src/filesystem/hl_operations/hl_readdir.js
View File

@ -81,13 +81,6 @@ class HLReadDir extends HLFilesystemOperation {
await child.fetchSuggestedApps(user);
await child.fetchSubdomains(user);
}
const fs = require('fs');
// fs.appendFileSync('/tmp/children.log',
// JSON.stringify({
// no_thumbs,
// no_assocs,
// entry: child.entry,
// }) + '\n');
return await child.getSafeEntry({ thumbnail: ! no_thumbs });
}));
}

8
src/backend/src/filesystem/hl_operations/hl_write.js
View File

@ -23,7 +23,7 @@ const StringParam = require("../../api/filesystem/StringParam");
const UserParam = require("../../api/filesystem/UserParam");
const config = require("../../config");
const { chkperm, validate_fsentry_name } = require("../../helpers");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require("@heyputer/putility").libs.promise;
const { pausing_tee, logging_stream, offset_write_stream, stream_to_the_void } = require("../../util/streamutil");
const { TYPE_DIRECTORY } = require("../FSNodeContext");
const { LLRead } = require("../ll_operations/ll_read");
@ -65,8 +65,7 @@ class WriteCommonFeature {
if ( ! user ) user = this.values.actor.type.user;
const usage = await sizeService.get_usage(user.id);
let capacity = config.is_storage_limited ? user.free_storage == undefined
? config.storage_capacity : user.free_storage : config.available_device_storage;
const capacity = await sizeService.get_storage_capacity(user.id);
if( capacity - usage - file.size < 0 ) {
throw APIError.create('storage_limit_reached');
}
@ -116,7 +115,6 @@ class HLWrite extends HLFilesystemOperation {
static MODULES = {
_path: require('path'),
socketio: require('../../socketio.js'),
mime: require('mime-types'),
}
@ -303,7 +301,7 @@ class HLWrite extends HLFilesystemOperation {
this.checkpoint('before thumbnail');
let thumbnail_promise = new TeePromise();
if ( await destination.isAppDataDirectory() ) {
if ( await destination.isAppDataDirectory() || values.no_thumbnail ) {
thumbnail_promise.resolve(undefined);
} else (async () => {
const reason = await (async () => {

35
src/backend/src/filesystem/ll_operations/ll_copy.js
View File

@ -21,7 +21,7 @@ const { Context } = require('../../util/context');
const { ParallelTasks } = require('../../util/otelutil');
const FSNodeContext = require('../FSNodeContext');
const { NodeUIDSelector } = require('../node/selectors');
const { RESOURCE_STATUS_PENDING_CREATE } = require('../storage/ResourceService');
const { RESOURCE_STATUS_PENDING_CREATE } = require('../../modules/puterfs/ResourceService');
const { UploadProgressTracker } = require('../storage/UploadProgressTracker');
const { LLFilesystemOperation } = require('./definitions');
@ -157,15 +157,15 @@ class LLCopy extends LLFilesystemOperation {
status: RESOURCE_STATUS_PENDING_CREATE,
});
const svc_fsentry = svc.get('systemFSEntryService');
const svc_fsEntry = svc.get('fsEntryService');
this.log.info(`inserting entry: ` + uuid);
const entryOp = await svc_fsentry.insert(raw_fsentry);
const entryOp = await svc_fsEntry.insert(raw_fsentry);
let node;
this.checkpoint('before parallel tasks');
const tasks = new ParallelTasks({ tracer, max: 4 });
await tracer.startActiveSpan(`fs:cp:parallel-portion`, async span => {
await Context.arun(`fs:cp:parallel-portion`, async () => {
this.checkpoint('starting parallel tasks');
// Add child copy tasks if this is a directory
if ( source.entry.is_dir ) {
@ -181,7 +181,6 @@ class LLCopy extends LLFilesystemOperation {
const child_name = await child_node.get('name');
// TODO: this should be LLCopy instead
const ll_copy = new LLCopy();
console.log('LL Copy Start');
await ll_copy.run({
source: await fs.node(
new NodeUIDSelector(child_uuid)
@ -192,30 +191,6 @@ class LLCopy extends LLFilesystemOperation {
user,
target_name: child_name,
});
console.log('LL Copy End');
// const hl_copy = new HLCopy();
// await hl_copy.run({
// destination_or_parent: await fs.node(
// new NodeUIDSelector(uuid)
// ),
// source: await fs.node(
// new NodeUIDSelector(child_uuid)
// ),
// user
// });
// await fs.cp(fs, {
// source: await fs.node(
// new NodeUIDSelector(child_uuid)
// ),
// // TODO: don't do this when cp supports uuids
// destinationOrParent: await fs.node(
// new NodeUIDSelector(uuid)
// ),
// user,
// overwrite: false,
// create_missing_parents: false,
// ancestor_check_not_needed: true,
// });
});
}
}
@ -241,8 +216,6 @@ class LLCopy extends LLFilesystemOperation {
this.checkpoint('waiting for parallel tasks');
await tasks.awaitAll();
this.checkpoint('finishing up');
span.end();
});
node = node || await fs.node(new NodeUIDSelector(uuid));

6
src/backend/src/filesystem/ll_operations/ll_mkdir.js
View File

@ -21,7 +21,7 @@ const { MODE_WRITE } = require("../../services/fs/FSLockService");
const { Context } = require("../../util/context");
const { TYPE_DIRECTORY } = require("../FSNodeContext");
const { NodeUIDSelector, NodeChildSelector } = require("../node/selectors");
const { RESOURCE_STATUS_PENDING_CREATE } = require("../storage/ResourceService");
const { RESOURCE_STATUS_PENDING_CREATE } = require("../../modules/puterfs/ResourceService");
const { LLFilesystemOperation } = require("./definitions");
class LLMkdir extends LLFilesystemOperation {
@ -60,7 +60,7 @@ class LLMkdir extends LLFilesystemOperation {
const ts = Math.round(Date.now() / 1000);
const uid = uuidv4();
const resourceService = context.get('services').get('resourceService');
const systemFSEntryService = context.get('services').get('systemFSEntryService');
const svc_fsEntry = context.get('services').get('fsEntryService');
const svc_event = context.get('services').get('event');
const fs = context.get('services').get('filesystem');
@ -109,7 +109,7 @@ class LLMkdir extends LLFilesystemOperation {
this.log.debug('creating fsentry', { fsentry: raw_fsentry })
this.checkpoint('about to enqueue insert');
const entryOp = await systemFSEntryService.insert(raw_fsentry);
const entryOp = await svc_fsEntry.insert(raw_fsentry);
this.field('fsentry-created', false);

1
src/backend/src/filesystem/ll_operations/ll_read.js
View File

@ -17,7 +17,6 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const APIError = require("../../api/APIError");
const { CodeModel } = require("../../codex/CodeModel");
const { Sequence } = require("../../codex/Sequence");
const { DB_WRITE } = require("../../services/database/consts");

2
src/backend/src/filesystem/ll_operations/ll_readdir.js
View File

@ -25,7 +25,7 @@ const { LLFilesystemOperation } = require("./definitions");
class LLReadDir extends LLFilesystemOperation {
async _run () {
const { context } = this;
const { subject: subject_let, user, actor, no_acl } = this.values;
const { subject: subject_let, actor, no_acl } = this.values;
let subject = subject_let;
if ( ! await subject.exists() ) {

2
src/backend/src/filesystem/ll_operations/ll_readshares.js
View File

@ -36,7 +36,7 @@ class LLReadShares extends LLFilesystemOperation {
`;
async _run () {
const { subject, user, actor, depth = 0 } = this.values;
const { subject, user, actor } = this.values;
const svc = this.context.get('services');

10
src/backend/src/filesystem/ll_operations/ll_write.js
View File

@ -18,7 +18,7 @@
*/
const { Context } = require("../../util/context");
const { LLFilesystemOperation } = require("./definitions");
const { RESOURCE_STATUS_PENDING_CREATE } = require("../storage/ResourceService");
const { RESOURCE_STATUS_PENDING_CREATE } = require("../../modules/puterfs/ResourceService.js");
const { NodeUIDSelector } = require("../node/selectors");
const { UploadProgressTracker } = require("../storage/UploadProgressTracker");
const FSNodeContext = require("../FSNodeContext");
@ -136,7 +136,7 @@ class LLOWrite extends LLWriteBase {
const svc = Context.get('services');
const sizeService = svc.get('sizeService');
const resourceService = svc.get('resourceService');
const systemFSEntryService = svc.get('systemFSEntryService');
const svc_fsEntry = svc.get('fsEntryService');
const svc_event = svc.get('event');
// TODO: fs:decouple-versions
@ -188,7 +188,7 @@ class LLOWrite extends LLWriteBase {
const filesize = file.size;
sizeService.change_usage(actor.type.user.id, filesize);
const entryOp = await systemFSEntryService.update(uid, raw_fsentry_delta);
const entryOp = await svc_fsEntry.update(uid, raw_fsentry_delta);
// depends on fsentry, does not depend on S3
(async () => {
@ -235,7 +235,7 @@ class LLCWrite extends LLWriteBase {
const svc = Context.get('services');
const sizeService = svc.get('sizeService');
const resourceService = svc.get('resourceService');
const systemFSEntryService = svc.get('systemFSEntryService');
const svc_fsEntry = svc.get('fsEntryService');
const svc_event = svc.get('event');
const fs = svc.get('filesystem');
@ -317,7 +317,7 @@ class LLCWrite extends LLWriteBase {
this.checkpoint('after change_usage');
const entryOp = await systemFSEntryService.insert(raw_fsentry);
const entryOp = await svc_fsEntry.insert(raw_fsentry);
this.checkpoint('after fsentry insert enqueue');

165
src/backend/src/filesystem/storage/SystemFSEntryService.js
View File

@ -1,165 +0,0 @@
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { PuterPath } = require("../lib/PuterPath");
const _path = require('path');
// Redis keys:
// <env>:<service>:<class>:<type>:<property>:<id>
//
// note: <environment> is added by redisService automatically.
//
// If `<type>` is `multi`, then the format differs slightly:
// <env>:<service>:<class>:multi:<type>:<property>:<id-property>:<id>
// where `<id-property>` specifies the property being used for the id
class SystemFSEntryService {
constructor ({ services }) {
this.redis = { enabled: false };
this.DatabaseFSEntryService = services.get('fsEntryService');
this.log = services.get('log-service').create('system-fsentry-service');
// Register information providers
const info = services.get('information');
this.info = info;
if ( ! this.redis.enabled ) return;
// path -> uuid via redis
info.given('fs.fsentry:path').provide('fs.fsentry:uuid')
.addStrategy('redis', async path => {
return await this.get_uuid_from_path(path);
});
// uuid -> path via redis
info.given('fs.fsentry:uuid').provide('fs.fsentry:path')
.addStrategy('redis', async uuid => {
this.log.debug('getting path for: ' + uuid);
if ( uuid === PuterPath.NULL_UUID ) return '/';
const res = ( await this.redis.get(`fs:fsentry:path:path:${uuid}`) ) ?? undefined;
this.log.debug('got path: ' + res);
return res;
});
// uuid -> parent_uuid via redis
info.given('fs.fsentry:uuid').provide('fs.fsentry:children(fs.fsentry:uuid)')
.addStrategy('redis', async uuid => {
return await this.get_child_uuids(uuid);
});
}
async insert (entry) {
if ( this.redis.enabled ) {
await this._link(entry.uuid, entry.parent_uid, entry.name);
}
return await this.DatabaseFSEntryService.insert(entry);
}
async update (uuid, entry) {
// If parent_uid is set during an update, we assume that it
// has been changed. If it hasn't, no problem: just an extra
// cache invalidation; but the code that set it should know
// better because it probably has the fsentry data already.
if ( entry.hasOwnProperty('parent_uid') ) {
await this._relocate(uuid, entry.parent_uid)
}
return await this.DatabaseFSEntryService.update(uuid, entry);
}
async delete (uuid) {
//
}
async get_child_uuids (uuid) {
let members;
members = await this.redis.smembers(`fs:fsentry:set:childs:${uuid}`);
if ( members ) return members;
members = await this.DatabaseFSEntryService.get_descendants(uuid);
return members ?? [];
}
async get_uuid_from_path (path) {
path = PuterPath.adapt(path);
let current = path.reference;
let pathOfReference = path.reference === PuterPath.NULL_UUID
? '/' : this.get_path_from_uuid(path.reference);
const fullPath = _path.join(pathOfReference, path.relativePortion);
let uuid = await this.redis.get(`fs:fsentry:multi:uuid:uuid:path:${fullPath}`);
return uuid;
}
// Cache related functions
async _link (subject_uuid, parent_uuid, subject_name) {
this.log.info(`linking ${subject_uuid} to ${parent_uuid}`);
// We need the parent's path to update everything
let pathOfParent = await this.info.with('fs.fsentry:uuid')
.obtain('fs.fsentry:path').exec(parent_uuid);
this.log.debug(`path of parent: ${pathOfParent}`);
if ( ! subject_name ) {
subject_name = await this.redis.get(`fs:fsentry:str:name:${subject_uuid}`);
}
// Register properties
await this.redis.set(`fs:fsentry:uuid:parent:${subject_uuid}`, parent_uuid);
await this.redis.set(`fs:fsentry:str:name:${subject_uuid}`, subject_name);
// Add as child of parent
await this.redis.sadd(`fs:fsentry:set:childs:${parent_uuid}`, subject_uuid);
// Register path
const subject_path = `${pathOfParent}/${subject_name}`;
this.log.debug(`registering path: ${subject_path} for ${subject_uuid}`);
await this.redis.set(`fs:fsentry:path:path:${subject_uuid}`, subject_path);
await this.redis.set(`fs:fsentry:multi:uuid:uuid:path:${subject_path}`, subject_uuid);
}
async _unlink (subject_uuid) {
let parent_uuid = await this.redis.get(`fs:fsentry:uuid:parent:${subject_uuid}`);
// TODO: try getting from database
// Remove from parent
await this.redis.srem(`fs:fsentry:set:childs:${parent_uuid}`, subject_uuid);
}
async _purge (subject_uuid) {
await this._unlink(subject_uuid);
// Remove properties
await this.redis.del(`fs:fsentry:uuid:parent:${subject_uuid}`);
await this.redis.del(`fs:fsentry:str:name:${subject_uuid}`);
// Remove path
const subject_path =
await this.redis.get(`fs:fsentry:path:path:${subject_uuid}`);
await this.redis.del(`fs:fsentry:path:path:${subject_uuid}`);
if ( subject_path ) {
await this.redis.del(`fs:fsentry:multi:uuid:path:${subject_path}`);
}
}
async _relocate (subject_uuid, new_parent_uuid) {
await this._unlink(subject_uuid);
await this._link(subject_uuid, new_parent_uuid);
}
}
module.exports = SystemFSEntryService;

2
src/backend/src/fun/dev-console-ui-utils.js
View File

@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const config = require('../config');
const { TeePromise } = require('../util/promise');
const { TeePromise } = require('@heyputer/putility').libs.promise;
const es_import_promise = new TeePromise();
let stringLength;

478
src/backend/src/helpers.js
View File

@ -22,7 +22,6 @@ const micromatch = require('micromatch');
const config = require('./config')
const mime = require('mime-types');
const PerformanceMonitor = require('./monitor/PerformanceMonitor.js');
const { generate_identifier } = require('./util/identifier.js');
const { ManagedError } = require('./util/errorutil.js');
const { spanify } = require('./util/otelutil.js');
const APIError = require('./api/APIError.js');
@ -1012,7 +1011,6 @@ async function gen_public_token(file_uuid, ttl = 24 * 60 * 60){
}
const uid = fsentry.uuid;
const expires = Math.ceil(Date.now() / 1000) + ttl;
const token = uuidv4();
const contentType = mime.contentType(fsentry.name);
@ -1117,6 +1115,7 @@ async function jwt_auth(req){
}
return {
actor,
user: actor.type.user,
token: token,
};
@ -1155,201 +1154,6 @@ async function jwt_auth(req){
return ancestors;
}
// THIS LEGACY FUNCTION IS STILL IN USE
// by: generate_system_fsentries
// TODO: migrate generate_system_fsentries to use QuickMkdir
async function mkdir(options){
const fs = systemfs;
debugger;
const resolved_path = PathBuilder.resolve(options.path, { puterfs: true });
const dirpath = _path.dirname(resolved_path);
let target_name = _path.basename(resolved_path);
const overwrite = options.overwrite ?? false;
const dedupe_name = options.dedupe_name ?? false;
const immutable = options.immutable ?? false;
const return_id = options.return_id ?? false;
const no_perm_check = options.no_perm_check ?? false;
// make parent directories as needed
const create_missing_parents = options.create_missing_parents ?? false;
// hold a list of all parent directories created in the process
let parent_dirs_created = [];
let overwritten_uid;
// target_name validation
try{
validate_fsentry_name(target_name)
}catch(e){
throw e.message;
}
// resolve dirpath to its fsentry
let parent = await convert_path_to_fsentry(dirpath);
// dirpath not found
if(parent === false && !create_missing_parents)
throw new Error("Target path not found");
// create missing parent directories
else if(parent === false && create_missing_parents){
const dirs = _path.resolve('/', dirpath).split('/');
let cur_path = '';
for(let j=0; j < dirs.length; j++){
if(dirs[j] === '')
continue;
cur_path += '/'+dirs[j];
// skip creating '/[username]'
if(j === 1)
continue;
try{
let d = await mkdir(fs, {path: cur_path, user: options.user});
d.path = cur_path;
parent_dirs_created.push(d);
}catch(e){
console.log(`Skipped mkdir ${cur_path}`);
}
}
// try setting parent again
parent = await convert_path_to_fsentry(dirpath);
if(parent === false)
throw new Error("Target path not found");
}
// check permission
if(!no_perm_check && !await chkperm(parent, options.user.id, 'write'))
throw { code:`forbidden`, message: `permission denied.`};
// check if a fsentry with the same name exists under this path
const existing_fsentry = await convert_path_to_fsentry(_path.resolve('/', dirpath + '/' + target_name ));
/** @type BaseDatabaseAccessService */
const db = services.get('database').get(DB_WRITE, 'filesystem');
// if trying to create a directory with an existing path and overwrite==false, throw an error
if(!overwrite && !dedupe_name && existing_fsentry !== false){
throw {
code: 'path_exists',
message:"A file/directory with the same path already exists.",
entry_name: existing_fsentry.name,
existing_fsentry: {
name: existing_fsentry.name,
uid: existing_fsentry.uuid,
}
};
}
else if(overwrite && existing_fsentry){
overwritten_uid = existing_fsentry.uuid;
// check permission
if(!await chkperm(existing_fsentry, options.user.id, 'write'))
throw {code:`forbidden`, message: `permission denied.`};
// delete existing dir
await db.write(
`DELETE FROM fsentries WHERE id = ? AND user_id = ?`,
[
//parent_uid
existing_fsentry.uuid,
//user_id
options.user.id,
]);
}
// dedupe name, generate a new name until its unique
else if(dedupe_name && existing_fsentry !== false){
for( let i = 1; ; i++){
let try_new_name = existing_fsentry.name + ' (' + i + ')';
let check_dupe = await db.read(
"SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1",
[existing_fsentry.parent_uid, try_new_name]
);
if(check_dupe[0] === undefined){
target_name = try_new_name;
break;
}
}
}
// shrotcut?
let shortcut_fsentry;
if(options.shortcut_to){
shortcut_fsentry = await uuid2fsentry(options.shortcut_to);
if(shortcut_fsentry === false){
throw ({ code:`not_found`, message: `shortcut_to not found.`})
}else if(!parent.is_dir){
throw ({ code:`not_dir`, message: `parent of shortcut_to must be a directory`})
}else if(!await chkperm(shortcut_fsentry, options.user.id, 'read')){
throw ({ code:`forbidden`, message: `shortcut_to permission denied.`})
}
}
// current epoch
const ts = Math.round(Date.now() / 1000)
const uid = uuidv4();
// record in db
let user_id = (parent === null ? options.user.id : parent.user_id);
const { insertId: mkdir_db_id } = await db.write(
`INSERT INTO fsentries
(uuid, parent_uid, user_id, name, is_dir, created, modified, immutable, shortcut_to, is_shortcut) VALUES
( ?, ?, ?, ?, true, ?, ?, ?, ?, ?)`,
[
//uuid
uid,
//parent_uid
(parent === null) ? null : parent.uuid,
//user_id
user_id,
//name
target_name,
//created
ts,
//modified
ts,
//immutable
immutable,
//shortcut_to,
shortcut_fsentry ? shortcut_fsentry.id : null,
//is_shortcut,
shortcut_fsentry ? 1 : 0,
]
);
const ret_obj = {
uid : uid,
name: target_name,
immutable: immutable,
is_dir: true,
path: options.path ?? false,
dirpath: dirpath,
is_shared: await is_shared_with_anyone(mkdir_db_id),
overwritten_uid: overwritten_uid,
shortcut_to: shortcut_fsentry ? shortcut_fsentry.uuid : null,
shortcut_to_path: shortcut_fsentry ? await id2path(shortcut_fsentry.id) : null,
parent_dirs_created: parent_dirs_created,
original_client_socket_id: options.original_client_socket_id,
};
// add existing_fsentry if exists
if(existing_fsentry){
ret_obj.existing_fsentry ={
name: existing_fsentry.name,
uid: existing_fsentry.uuid,
}
}
if(return_id)
ret_obj.id = mkdir_db_id;
// send realtime success msg to client
let socketio = require('./socketio.js').getio();
if(socketio){
socketio.to(user_id).emit('item.added', ret_obj)
}
return ret_obj;
}
function is_valid_uuid ( uuid ) {
let s = "" + uuid;
s = s.match(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i);
@ -1412,100 +1216,6 @@ async function app_name_exists(name){
return true;
}
// generates all the default files and directories a user needs,
// generally used for a brand new account
async function generate_system_fsentries(user){
/** @type BaseDatabaseAccessService */
const db = services.get('database').get(DB_WRITE, 'filesystem');
//-------------------------------------------------------------
// create root `/[username]/`
//-------------------------------------------------------------
const root_dir = await mkdir({
path: '/' + user.username,
user: user,
immutable: true,
no_perm_check: true,
return_id: true,
});
// Normally, it is recommended to use mkdir() to create new folders,
// but during signup this could result in multiple queries to the DB server
// and for servers in remote regions such as Asia this could result in a
// very long time for /signup to finish, sometimes up to 30-40 seconds!
// by combining as many queries as we can into one and avoiding multiple back-and-forth
// with the DB server, we can speed this process up significantly.
const ts = Date.now()/1000;
// Generate UUIDs for all the default folders and files
let trash_uuid = uuidv4();
let appdata_uuid = uuidv4();
let desktop_uuid = uuidv4();
let documents_uuid = uuidv4();
let pictures_uuid = uuidv4();
let videos_uuid = uuidv4();
let public_uuid = uuidv4();
const insert_res = await db.write(
`INSERT INTO fsentries
(uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable) VALUES
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true),
( ?, ?, ?, ?, ?, true, ?, ?, true)
`,
[
// Trash
trash_uuid, root_dir.uid, user.id, 'Trash', `/${user.username}/Trash`, ts, ts,
// AppData
appdata_uuid, root_dir.uid, user.id, 'AppData', `/${user.username}/AppData`, ts, ts,
// Desktop
desktop_uuid, root_dir.uid, user.id, 'Desktop', `/${user.username}/Desktop`, ts, ts,
// Documents
documents_uuid, root_dir.uid, user.id, 'Documents', `/${user.username}/Documents`, ts, ts,
// Pictures
pictures_uuid, root_dir.uid, user.id, 'Pictures', `/${user.username}/Pictures`, ts, ts,
// Videos
videos_uuid, root_dir.uid, user.id, 'Videos', `/${user.username}/Videos`, ts, ts,
// Public
public_uuid, root_dir.uid, user.id, 'Public', `/${user.username}/Public`, ts, ts,
]
);
// https://stackoverflow.com/a/50103616
let trash_id = insert_res.insertId;
let appdata_id = insert_res.insertId + 1;
let desktop_id = insert_res.insertId + 2;
let documents_id = insert_res.insertId + 3;
let pictures_id = insert_res.insertId + 4;
let videos_id = insert_res.insertId + 5;
let public_id = insert_res.insertId + 6;
// Asynchronously set the user's system folders uuids in database
// This is for caching purposes, so we don't have to query the DB every time we need to access these folders
// This is also possible because we know the user's system folders uuids will never change
// TODO: pass to IIAFE manager to avoid unhandled promise rejection
// (IIAFE manager doesn't exist yet, hence this is a TODO)
db.write(
`UPDATE user SET
trash_uuid=?, appdata_uuid=?, desktop_uuid=?, documents_uuid=?, pictures_uuid=?, videos_uuid=?, public_uuid=?,
trash_id=?, appdata_id=?, desktop_id=?, documents_id=?, pictures_id=?, videos_id=?, public_id=?
WHERE id=?`,
[
trash_uuid, appdata_uuid, desktop_uuid, documents_uuid, pictures_uuid, videos_uuid, public_uuid,
trash_id, appdata_id, desktop_id, documents_id, pictures_id, videos_id, public_id,
user.id
]
);
invalidate_cached_user(user);
}
function send_email_verification_code(email_confirm_code, email){
const svc_email = Context.get('services').get('email');
svc_email.send_email({ email }, 'email_verification_code', {
@ -1519,19 +1229,11 @@ function send_email_verification_token(email_confirm_token, email, user_uuid){
svc_email.send_email({ email }, 'email_verification_link', { link });
}
async function generate_random_username(){
let username;
do {
username = generate_identifier();
} while (await username_exists(username));
return username;
}
function generate_random_str(length) {
var result = '';
var characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
var charactersLength = characters.length;
for ( var i = 0; i < length; i++ ) {
let result = '';
const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
const charactersLength = characters.length;
for ( let i = 0; i < length; i++ ) {
result += characters.charAt(Math.floor(Math.random() *
charactersLength));
}
@ -1546,11 +1248,11 @@ function generate_random_str(length) {
* @throws {TypeError} If the `seconds` parameter is not a number.
*/
function seconds_to_string(seconds) {
var numyears = Math.floor(seconds / 31536000);
var numdays = Math.floor((seconds % 31536000) / 86400);
var numhours = Math.floor(((seconds % 31536000) % 86400) / 3600);
var numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60);
var numseconds = (((seconds % 31536000) % 86400) % 3600) % 60;
const numyears = Math.floor(seconds / 31536000);
const numdays = Math.floor((seconds % 31536000) / 86400);
const numhours = Math.floor(((seconds % 31536000) % 86400) / 3600);
const numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60);
const numseconds = (((seconds % 31536000) % 86400) % 3600) % 60;
return numyears + " years " + numdays + " days " + numhours + " hours " + numminutes + " minutes " + numseconds + " seconds";
}
@ -1565,14 +1267,11 @@ async function suggest_app_for_fsentry(fsentry, options){
const suggested_apps = [];
let content_type = mime.contentType(fsentry.name);
if(content_type === null || content_type === undefined || content_type === false)
content_type = '';
if( ! content_type ) content_type = '';
// IIFE just so fsname can stay `const`
const fsname = (() => {
if ( ! fsentry.name ) {
const fs = require('fs');
fs.writeFileSync('/tmp/missing-fsentry-name.txt', JSON.stringify(fsentry, null, 2));
return 'missing-fsentry-name';
}
let fsname = fsentry.name.toLowerCase();
@ -1582,73 +1281,78 @@ async function suggest_app_for_fsentry(fsentry, options){
})();
const file_extension = _path.extname(fsname).toLowerCase();
const any_of = (list, name) => {
return list.some(v => name.endsWith(v));
}
//---------------------------------------------
// Code
//---------------------------------------------
if(
fsname.endsWith('.asm') ||
fsname.endsWith('.asp') ||
fsname.endsWith('.aspx') ||
fsname.endsWith('.bash') ||
fsname.endsWith('.c') ||
fsname.endsWith('.cpp') ||
fsname.endsWith('.css') ||
fsname.endsWith('.csv') ||
fsname.endsWith('.dhtml') ||
fsname.endsWith('.f') ||
fsname.endsWith('.go') ||
fsname.endsWith('.h') ||
fsname.endsWith('.htm') ||
fsname.endsWith('.html') ||
fsname.endsWith('.html5') ||
fsname.endsWith('.java') ||
fsname.endsWith('.jl') ||
fsname.endsWith('.js') ||
fsname.endsWith('.jsa') ||
fsname.endsWith('.json') ||
fsname.endsWith('.jsonld') ||
fsname.endsWith('.jsf') ||
fsname.endsWith('.jsp') ||
fsname.endsWith('.kt') ||
fsname.endsWith('.log') ||
fsname.endsWith('.lock') ||
fsname.endsWith('.lua') ||
fsname.endsWith('.md') ||
fsname.endsWith('.perl') ||
fsname.endsWith('.phar') ||
fsname.endsWith('.php') ||
fsname.endsWith('.pl') ||
fsname.endsWith('.py') ||
fsname.endsWith('.r') ||
fsname.endsWith('.rb') ||
fsname.endsWith('.rdata') ||
fsname.endsWith('.rda') ||
fsname.endsWith('.rdf') ||
fsname.endsWith('.rds') ||
fsname.endsWith('.rs') ||
fsname.endsWith('.rlib') ||
fsname.endsWith('.rpy') ||
fsname.endsWith('.scala') ||
fsname.endsWith('.sc') ||
fsname.endsWith('.scm') ||
fsname.endsWith('.sh') ||
fsname.endsWith('.sol') ||
fsname.endsWith('.sql') ||
fsname.endsWith('.ss') ||
fsname.endsWith('.svg') ||
fsname.endsWith('.swift') ||
fsname.endsWith('.toml') ||
fsname.endsWith('.ts') ||
fsname.endsWith('.wasm') ||
fsname.endsWith('.xhtml') ||
fsname.endsWith('.xml') ||
fsname.endsWith('.yaml') ||
// files with no extension
!fsname.includes('.')
){
const exts_code = [
'.asm',
'.asp',
'.aspx',
'.bash',
'.c',
'.cpp',
'.css',
'.csv',
'.dhtml',
'.f',
'.go',
'.h',
'.htm',
'.html',
'.html5',
'.java',
'.jl',
'.js',
'.jsa',
'.json',
'.jsonld',
'.jsf',
'.jsp',
'.kt',
'.log',
'.lock',
'.lua',
'.md',
'.perl',
'.phar',
'.php',
'.pl',
'.py',
'.r',
'.rb',
'.rdata',
'.rda',
'.rdf',
'.rds',
'.rs',
'.rlib',
'.rpy',
'.scala',
'.sc',
'.scm',
'.sh',
'.sol',
'.sql',
'.ss',
'.svg',
'.swift',
'.toml',
'.ts',
'.wasm',
'.xhtml',
'.xml',
'.yaml',
];
if ( any_of(exts_code, fsname) || !fsname.includes('.') ) {
suggested_apps.push(await get_app({name: 'code'}))
suggested_apps.push(await get_app({name: 'editor'}))
}
//---------------------------------------------
// Editor
//---------------------------------------------
@ -1712,20 +1416,18 @@ async function suggest_app_for_fsentry(fsentry, options){
//---------------------------------------------
// 3rd-party apps
//---------------------------------------------
const apps = kv.get(`assocs:${file_extension.slice(1)}:apps`)
const apps = kv.get(`assocs:${file_extension.slice(1)}:apps`) ?? [];
monitor.label("third party associations");
if(apps && apps.length > 0){
for (let index = 0; index < apps.length; index++) {
for ( const app_id of apps ) {
// retrieve app from DB
const third_party_app = await get_app({id: apps[index]})
const third_party_app = await get_app({id: app_id})
if ( ! third_party_app ) continue;
// only add if the app is approved for opening items or the app is owned by this user
if( third_party_app.approved_for_opening_items ||
(options !== undefined && options.user !== undefined && options.user.id === third_party_app.owner_user_id))
suggested_apps.push(third_party_app)
}
}
monitor.stamp();
monitor.end();
@ -1741,10 +1443,6 @@ async function suggest_app_for_fsentry(fsentry, options){
});
}
function build_item_object(item){
}
async function get_taskbar_items(user) {
/** @type BaseDatabaseAccessService */
const db = services.get('database').get(DB_WRITE, 'filesystem');
@ -1813,7 +1511,7 @@ async function get_taskbar_items(user) {
return taskbar_items;
}
function validate_signature_auth(url, action) {
function validate_signature_auth(url, action, options = {}) {
const query = new URL(url).searchParams;
if(!query.get('uid'))
@ -1825,6 +1523,12 @@ function validate_signature_auth(url, action) {
else if(!query.get('signature'))
throw {message: '`signature` is required for signature-based authentication.'}
if ( options.uid ) {
if ( query.get('uid') !== options.uid ) {
throw {message: 'Authentication failed. `uid` does not match.'}
}
}
const expired = query.get('expires') && (query.get('expires') < Date.now() / 1000);
// expired?
@ -1867,13 +1571,13 @@ async function mv(options){
function number_format (number, decimals, dec_point, thousands_sep) {
// Strip all characters but numerical ones.
number = (number + '').replace(/[^0-9+\-Ee.]/g, '');
var n = !isFinite(+number) ? 0 : +number,
let n = !isFinite(+number) ? 0 : +number,
prec = !isFinite(+decimals) ? 0 : Math.abs(decimals),
sep = (typeof thousands_sep === 'undefined') ? ',' : thousands_sep,
dec = (typeof dec_point === 'undefined') ? '.' : dec_point,
s = '',
toFixedFix = function (n, prec) {
var k = Math.pow(10, prec);
const k = Math.pow(10, prec);
return '' + Math.round(n * k) / k;
};
// Fix for IE parseFloat(0.55).toFixed(0) = 0;
@ -1893,7 +1597,6 @@ module.exports = {
app_name_exists,
app_exists,
body_parser_error_handler,
build_item_object,
byte_format,
change_username,
chkperm,
@ -1905,9 +1608,7 @@ module.exports = {
gen_public_token,
get_taskbar_items,
get_url_from_req,
generate_system_fsentries,
generate_random_str,
generate_random_username,
get_app,
get_user,
invalidate_cached_user,
@ -1926,7 +1627,6 @@ module.exports = {
is_specifically_uuidv4,
is_valid_url,
jwt_auth,
mkdir,
mv,
number_format,
refresh_apps_cache,

1
src/backend/src/middleware/abuse.js
View File

@ -20,7 +20,6 @@ const APIError = require("../api/APIError");
const { Context } = require("../util/context");
const abuse = options => (req, res, next) => {
// const svc_abuse = x.get('services').get('abuse-prevention');
const requester = Context.get('requester');
if ( options.no_bots ) {

148
src/backend/src/services/runtime-analysis/AlarmService.js → src/backend/src/modules/core/AlarmService.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@ -23,55 +24,59 @@ const util = require('util');
const _path = require('path');
const fs = require('fs');
const { fallbackRead } = require('../../util/files.js');
const { generate_identifier } = require('../../util/identifier.js');
const { stringify_log_entry } = require('./LogService.js');
const BaseService = require('../BaseService.js');
const { split_lines } = require('../../util/stdioutil.js');
const { Context } = require('../../util/context.js');
const BaseService = require('../../services/BaseService.js');
/**
* AlarmService class is responsible for managing alarms.
* It provides methods for creating, clearing, and handling alarms.
*/
class AlarmService extends BaseService {
static USE = {
logutil: 'core.util.logutil',
identutil: 'core.util.identutil',
stdioutil: 'core.util.stdioutil',
Context: 'core.context',
}
/**
* This method initializes the AlarmService by setting up its internal data structures and initializing any required dependencies.
*
* It reads in the known errors from a JSON5 file and sets them as the known_errors property of the AlarmService instance.
*
* It also registers commands with the provided commands service.
*/
async _construct () {
this.alarms = {};
this.alarm_aliases = {};
this.known_errors = [];
}
/**
* Method to initialize AlarmService. Sets the known errors and registers commands.
* @returns {Promise<void>}
*/
async _init () {
const services = this.services;
this.pager = services.get('pager');
// TODO:[self-hosted] fix this properly
this.known_errors = [];
// (async () => {
// try {
// this.known_errors = JSON5.parse(
// await fallbackRead(
// 'data/known_errors.json5',
// '/var/puter/data/known_errors.json5',
// ),
// );
// } catch (e) {
// this.create(
// 'missing-known-errors',
// e.message,
// )
// }
// })();
this._register_commands(services.get('commands'));
if ( this.global_config.env === 'dev' ) {
/**
* This method initializes the AlarmService instance by registering commands, setting up the pager, and initializing the known errors.
* It also sets up the widget to display alarms in the dev environment.
*
* @param {BaseService} services - The BaseService instance that provides access to other services.
* @returns {void}
*/
this.alarm_widget = () => {
// return `\x1B[31;1m alarms (${
// Object.keys(this.alarms)
// })\x1B[0m`;
const lines = [];
for ( const alarm of Object.values(this.alarms) ) {
const line =
`\x1B[31;1m [alarm]\x1B[0m ` +
`${alarm.id_string}: ${alarm.message} (${alarm.count})`;
const line_lines = split_lines(line);
const line_lines = this.stdioutil.split_lines(line);
lines.push(...line_lines);
}
@ -80,26 +85,45 @@ class AlarmService extends BaseService {
}
}
adapt_id_ (id) {
// let shorten = false;
// // Check if id uses characters that aren't on a US QWERTY keyboard.
// if ( /[^\x20-\x7E]/.test(id) ) shorten = true;
/**
* AlarmService registers its commands at the consolidation phase because
* the '_init' method of CommandService may not have been called yet.
*/
['__on_boot.consolidation'] () {
this._register_commands(this.services.get('commands'));
}
// // Check if id is too long
// if ( id.length > 20 ) shorten = true;
adapt_id_ (id) {
let shorten = true;
if ( shorten ) {
const rng = seedrandom(id);
id = generate_identifier('-', rng);
id = this.identutil.generate_identifier('-', rng);
}
return id;
}
/**
* Method to create an alarm with the given ID, message, and fields.
* If the ID already exists, it will be updated with the new fields
* and the occurrence count will be incremented.
*
* @param {string} id - Unique identifier for the alarm.
* @param {string} message - Message associated with the alarm.
* @param {object} fields - Additional information about the alarm.
*/
create (id, message, fields) {
this.log.error(`upcoming alarm: ${id}: ${message}`);
let existing = false;
/**
* Method to create an alarm with the given ID, message, and fields.
* If the ID already exists, it will be updated with the new fields.
* @param {string} id - Unique identifier for the alarm.
* @param {string} message - Message associated with the alarm.
* @param {object} fields - Additional information about the alarm.
* @returns {void}
*/
const alarm = (() => {
const short_id = this.adapt_id_(id);
@ -116,12 +140,34 @@ class AlarmService extends BaseService {
};
Object.defineProperty(alarm, 'count', {
/**
* Method to create a new alarm.
*
* This method takes an id, message, and optional fields as parameters.
* It creates a new alarm object with the provided id and message,
* and adds it to the alarms object. It also keeps track of the number of occurrences of the alarm.
* If the alarm already exists, it increments the occurrence count and calls the handle\_alarm\_repeat\_ method.
* If it's a new alarm, it calls the handle\_alarm\_on\_ method.
*
* @param {string} id - The unique identifier for the alarm.
* @param {string} message - The message associated with the alarm.
* @param {object} [fields] - Optional fields associated with the alarm.
* @returns {void}
*/
get () {
return alarm.timestamps?.length ?? 0;
}
});
Object.defineProperty(alarm, 'id_string', {
/**
* Method to handle creating a new alarm with given parameters.
* This method adds the alarm to the `alarms` object, updates the occurrences count,
* and processes any known errors that may apply to the alarm.
* @param {string} id - The unique identifier for the alarm.
* @param {string} message - The message associated with the alarm.
* @param {Object} fields - Additional fields to associate with the alarm.
*/
get () {
if ( alarm.id.length < 20 ) {
return alarm.id;
@ -170,6 +216,11 @@ class AlarmService extends BaseService {
}
}
/**
* Method to clear an alarm with the given ID.
* @param {*} id - The ID of the alarm to clear.
* @returns {void}
*/
clear (id) {
const alarm = this.alarms[id];
if ( !alarm ) {
@ -252,7 +303,7 @@ class AlarmService extends BaseService {
svc_devConsole.add_widget(this.alarm_widget);
}
const args = Context.get('args') ?? {};
const args = this.Context.get('args') ?? {};
if ( args['quit-on-alarm'] ) {
const svc_shutdown = this.services.get('shutdown');
svc_shutdown.shutdown({
@ -284,7 +335,7 @@ class AlarmService extends BaseService {
// Write a .log file for the alert that happened
try {
const lines = [];
lines.push(`ALERT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`),
lines.push(`ALERT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`);
lines.push(`started: ${new Date(alarm.started).toISOString()}`);
lines.push(`short id: ${alarm.short_id}`);
lines.push(`original id: ${alarm.id}`);
@ -296,7 +347,7 @@ class AlarmService extends BaseService {
(async () => {
try {
await fs.appendFileSync(`alert_${alarm.id}.log`, alert_info + '\n');
fs.appendFileSync(`alert_${alarm.id}.log`, alert_info + '\n');
} catch (e) {
this.log.error(`failed to write alert log: ${e.message}`);
}
@ -313,11 +364,32 @@ class AlarmService extends BaseService {
);
}
/**
* Method to get an alarm by its ID.
*
* @param {*} id - The ID of the alarm to get.
* @returns
*/
get_alarm (id) {
return this.alarms[id] ?? this.alarm_aliases[id];
}
_register_commands (commands) {
// Function to handle a specific alarm event.
// This comment can be added above line 320.
// This function is responsible for processing specific events related to alarms.
// It can be used for tasks such as updating alarm status, sending notifications, or triggering actions.
// This function is called internally by the AlarmService class.
// /*
// * handleAlarmEvent - Handles a specific alarm event.
// *
// * @param {Object} alarm - The alarm object containing relevant information.
// * @param {Function} callback - Optional callback function to be called when the event is handled.
// */
// function handleAlarmEvent(alarm, callback) {
// // Implementation goes here.
// }
const completeAlarmID = (args) => {
// The alarm ID is the first argument, so return no results if we're on the second or later.
if (args.length > 1)
@ -424,7 +496,7 @@ class AlarmService extends BaseService {
}
log.log(`┏━━ Logs before: ${alarm.id_string} ━━━━`);
for ( const lg of occurance.logs ) {
log.log("┃ " + stringify_log_entry(lg));
log.log("┃ " + this.logutil.stringify_log_entry(lg));
}
log.log(`┗━━ Logs before: ${alarm.id_string} ━━━━`);
},

21
src/backend/src/modules/core/ContextService.js Normal file
View File

@ -0,0 +1,21 @@
const BaseService = require("../../services/BaseService");
const { Context } = require("../../util/context");
/**
* ContextService provides a way for other services to register a hook to be
* called when a context/subcontext is created.
*
* Contexts are used to provide contextual information in the execution
* context (dynamic scope). They can also be used to identify a "span";
* a span is a labelled frame of execution that can be used to track
* performance, errors, and other metrics.
*/
class ContextService extends BaseService {
register_context_hook (event, hook) {
Context.context_hooks_[event].push(hook);
}
}
module.exports = {
ContextService,
};

61
src/backend/src/modules/core/Core2Module.js Normal file
View File

@ -0,0 +1,61 @@
const { AdvancedBase } = require("@heyputer/putility");
/**
* A replacement for CoreModule with as few external relative requires as possible.
* This will eventually be the successor to CoreModule, the main module for Puter's backend.
*
* The scope of this module is:
* - logging and error handling
* - alarm handling
* - services that are tightly coupled with alarm handling are allowed
* - any essential information about server stats or health
* - any very generic service which other services can register
* behavior to.
*/
class Core2Module extends AdvancedBase {
async install (context) {
// === LIBS === //
const useapi = context.get('useapi');
const lib = require('./lib/__lib__.js');
for ( const k in lib ) {
useapi.def(`core.${k}`, lib[k], { assign: true });
}
useapi.def('core.context', require('../../util/context.js').Context);
// === SERVICES === //
const services = context.get('services');
const { LogService } = require('./LogService.js');
services.registerService('log-service', LogService);
const { AlarmService } = require("./AlarmService.js");
services.registerService('alarm', AlarmService);
const { ErrorService } = require("./ErrorService.js");
services.registerService('error-service', ErrorService);
const { PagerService } = require("./PagerService.js");
services.registerService('pager', PagerService);
const { ExpectationService } = require("./ExpectationService.js");
services.registerService('expectations', ExpectationService);
const { ProcessEventService } = require("./ProcessEventService.js");
services.registerService('process-event', ProcessEventService);
const { ServerHealthService } = require("./ServerHealthService.js");
services.registerService('server-health', ServerHealthService);
const { ParameterService } = require("./ParameterService.js");
services.registerService('params', ParameterService);
const { ContextService } = require('./ContextService.js');
services.registerService('context', ContextService);
}
}
module.exports = {
Core2Module,
};

49
src/backend/src/services/runtime-analysis/ErrorService.js → src/backend/src/modules/core/ErrorService.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@ -16,8 +17,20 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const BaseService = require("../BaseService");
const BaseService = require("../../services/BaseService");
/**
* **ErrorContext Class**
*
* The `ErrorContext` class is designed to encapsulate error reporting functionality within a specific logging context.
* It facilitates the reporting of errors by providing a method to log error details along with additional contextual information.
*
* @class
* @classdesc Provides a context for error reporting with specific logging details.
* @param {ErrorService} error_service - The error service instance to use for reporting errors.
* @param {object} log_context - The logging context to associate with the error reports.
*/
class ErrorContext {
constructor (error_service, log_context) {
this.error_service = error_service;
@ -32,15 +45,49 @@ class ErrorContext {
}
}
/**
* The ErrorService class is responsible for handling and reporting errors within the system.
* It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms.
* @class ErrorService
* @extends BaseService
*/
class ErrorService extends BaseService {
/**
* Initializes the ErrorService, setting up the alarm and backup logger services.
*
* @async
* @function init
* @memberof ErrorService
* @returns {Promise<void>} A promise that resolves when the initialization is complete.
*/
async init () {
const services = this.services;
this.alarm = services.get('alarm');
this.backupLogger = services.get('log-service').create('error-service');
}
/**
* Creates an ErrorContext instance with the provided logging context.
*
* @param {*} log_context The logging context to associate with the error reports.
* @returns {ErrorContext} An ErrorContext instance.
*/
create (log_context) {
return new ErrorContext(this, log_context);
}
/**
* Reports an error with the specified location and details.
* The "location" is a string up to the callers discretion to identify
* the source of the error.
*
* @param {*} location The location where the error occurred.
* @param {*} fields The error details to report.
* @param {boolean} [alarm=true] Whether to raise an alarm for the error.
* @returns {void}
*/
report (location, { source, logger, trace, extra, message }, alarm = true) {
message = message ?? source?.message;
logger = logger ?? this.backupLogger;

138
src/backend/src/modules/core/ExpectationService.js Normal file
View File

@ -0,0 +1,138 @@
// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { v4: uuidv4 } = require('uuid');
const BaseService = require('../../services/BaseService');
/**
* @class ExpectationService
* @extends BaseService
*
* The `ExpectationService` is a specialized service designed to assist in the diagnosis and
* management of errors related to the intricate interactions among asynchronous operations.
* It facilitates tracking and reporting on expectations, enabling better fault isolation
* and resolution in systems where synchronization and timing of operations are crucial.
*
* This service inherits from the `BaseService` and provides methods for registering,
* purging, and handling expectations, making it a valuable tool for diagnosing complex
* runtime behaviors in a system.
*/
class ExpectationService extends BaseService {
static USE = {
expect: 'core.expect'
};
/**
* Constructs the ExpectationService and initializes its internal state.
* This method is intended to be called asynchronously.
* It sets up the `expectations_` array which will be used to track expectations.
*
* @async
*/
async _construct () {
this.expectations_ = [];
}
/**
* ExpectationService registers its commands at the consolidation phase because
* the '_init' method of CommandService may not have been called yet.
*/
['__on_boot.consolidation'] () {
const commands = this.services.get('commands');
commands.registerCommands('expectations', [
{
id: 'pending',
description: 'lists pending expectations',
handler: async (args, log) => {
this.purgeExpectations_();
if ( this.expectations_.length < 1 ) {
log.log(`there are none`);
return;
}
for ( const expectation of this.expectations_ ) {
expectation.report(log);
}
}
}
]);
}
/**
* Initializes the ExpectationService, setting up interval functions and registering commands.
*
* This method sets up a periodic interval to purge expectations and registers a command
* to list pending expectations. The interval invokes `purgeExpectations_` every second.
* The command 'pending' allows users to list and log all pending expectations.
*
* @returns {Promise<void>} A promise that resolves when initialization is complete.
*/
async _init () {
// TODO: service to track all interval functions?
/**
* Initializes the service by setting up interval functions and registering commands.
* This method sets up a periodic interval function to purge expectations and registers
* a command to list pending expectations.
*
* @returns {void}
*/
// The comment should be placed above the method at line 68
setInterval(() => {
this.purgeExpectations_();
}, 1000);
}
/**
* Purges expectations that have been met.
*
* This method iterates through the list of expectations and removes
* those that have been satisfied. Currently, this functionality is
* disabled and needs to be re-enabled.
*
* @returns {void} This method does not return anything.
*/
purgeExpectations_ () {
return;
// TODO: Re-enable this
// for ( let i=0 ; i < this.expectations_.length ; i++ ) {
// if ( this.expectations_[i].check() ) {
// this.expectations_[i] = null;
// }
// }
// this.expectations_ = this.expectations_.filter(v => v !== null);
}
/**
* Registers an expectation to be tracked by the service.
*
* @param {Object} workUnit - The work unit to track
* @param {string} checkpoint - The checkpoint to expect
* @returns {void}
*/
expect_eventually ({ workUnit, checkpoint }) {
this.expectations_.push(new this.expect.CheckpointExpectation(workUnit, checkpoint));
}
}
module.exports = {
ExpectationService
};

178
src/backend/src/services/runtime-analysis/LogService.js → src/backend/src/modules/core/LogService.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"xai"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@ -27,7 +28,8 @@ const LOG_LEVEL_SYSTEM = logSeverity(4, 'SYSTEM', '33;1', 'system');
const winston = require('winston');
const { Context } = require('../../util/context');
const BaseService = require('../BaseService');
const BaseService = require('../../services/BaseService');
const { stringify_log_entry } = require('./lib/log');
require('winston-daily-rotate-file');
const WINSTON_LEVELS = {
@ -41,6 +43,13 @@ const WINSTON_LEVELS = {
silly: 60
};
/**
* Represents a logging context within the LogService.
* This class is used to manage logging operations with specific context information,
* allowing for hierarchical logging structures and dynamic field additions.
* @class LogContext
*/
class LogContext {
constructor (logService, { crumbs, fields }) {
this.logService = logService;
@ -96,8 +105,12 @@ class LogContext {
);
}
// convenience method to get a trace id that isn't as difficult
// for a human to read as a uuid.
/**
* Generates a human-readable trace ID for logging purposes.
*
* @returns {string} A trace ID in the format 'xxxxxx-xxxxxx' where each segment is a
* random string of six lowercase letters and digits.
*/
mkid () {
// generate trace id
const trace_id = [];
@ -107,55 +120,40 @@ class LogContext {
return trace_id.join('-');
}
// add a trace id to this logging context
/**
* Adds a trace id to this logging context for tracking purposes.
* @returns {LogContext} The current logging context with the trace id added.
*/
traceOn () {
this.fields.trace_id = this.mkid();
return this;
}
/**
* Gets the log buffer maintained by the LogService. This shows the most
* recent log entries.
* @returns {Array} An array of log entries stored in the buffer.
*/
get_log_buffer () {
return this.logService.get_log_buffer();
}
}
let log_epoch = Date.now();
const stringify_log_entry = ({ prefix, log_lvl, crumbs, message, fields, objects }) => {
const { colorize } = require('json-colorizer');
let lines = [], m;
const lf = () => {
if ( ! m ) return;
lines.push(m);
m = '';
}
m = prefix ? `${prefix} ` : '';
m += `\x1B[${log_lvl.esc}m[${log_lvl.label}\x1B[0m`;
for ( const crumb of crumbs ) {
m += `::${crumb}`;
}
m += `\x1B[${log_lvl.esc}m]\x1B[0m`;
if ( fields.timestamp ) {
// display seconds since logger epoch
const n = (fields.timestamp - log_epoch) / 1000;
m += ` (${n.toFixed(3)}s)`;
}
m += ` ${message} `;
lf();
for ( const k in fields ) {
if ( k === 'timestamp' ) continue;
let v; try {
v = colorize(JSON.stringify(fields[k]));
} catch (e) {
v = '' + fields[k];
}
m += ` \x1B[1m${k}:\x1B[0m ${v}`;
lf();
}
return lines.join('\n');
};
/**
* Timestamp in milliseconds since the epoch, used for calculating log entry duration.
*/
/**
* @class DevLogger
* @classdesc
* A development logger class designed for logging messages during development.
* This logger can either log directly to console or delegate logging to another logger.
* It provides functionality to turn logging on/off, and can optionally write logs to a file.
*
* @param {function} log - The logging function, typically `console.log` or similar.
* @param {object} [opt_delegate] - An optional logger to which log messages can be delegated.
*/
class DevLogger {
// TODO: this should eventually delegate to winston logger
constructor (log, opt_delegate) {
@ -195,6 +193,13 @@ class DevLogger {
}
}
/**
* @class NullLogger
* @description A logger that does nothing, effectively disabling logging.
* This class is used when logging is not desired or during development
* to avoid performance overhead or for testing purposes.
*/
class NullLogger {
// TODO: this should eventually delegate to winston logger
constructor (log, opt_delegate) {
@ -208,6 +213,12 @@ class NullLogger {
}
}
/**
* WinstonLogger Class
*
* A logger that delegates log messages to a Winston logger instance.
*/
class WinstonLogger {
constructor (winst) {
this.winst = winst;
@ -222,6 +233,15 @@ class WinstonLogger {
}
}
/**
* @class TimestampLogger
* @classdesc A logger that adds timestamps to log messages before delegating them to another logger.
* This class wraps another logger instance to ensure that all log messages include a timestamp,
* which can be useful for tracking the sequence of events in a system.
*
* @param {Object} delegate - The logger instance to which the timestamped log messages are forwarded.
*/
class TimestampLogger {
constructor (delegate) {
this.delegate = delegate;
@ -232,6 +252,15 @@ class TimestampLogger {
}
}
/**
* The `BufferLogger` class extends the logging functionality by maintaining a buffer of log entries.
* This class is designed to:
* - Store a specified number of recent log messages.
* - Allow for retrieval of these logs for debugging or monitoring purposes.
* - Ensure that the log buffer does not exceed the defined size by removing older entries when necessary.
* - Delegate logging messages to another logger while managing its own buffer.
*/
class BufferLogger {
constructor (size, delegate) {
this.size = size;
@ -247,6 +276,14 @@ class BufferLogger {
}
}
/**
* Represents a custom logger that can modify log messages before they are passed to another logger.
* @class CustomLogger
* @extends {Object}
* @param {Object} delegate - The delegate logger to which modified log messages will be passed.
* @param {Function} callback - A callback function that modifies log parameters before delegation.
*/
class CustomLogger {
constructor (delegate, callback) {
this.delegate = delegate;
@ -279,17 +316,39 @@ class CustomLogger {
}
}
/**
* The `LogService` class extends `BaseService` and is responsible for managing and
* orchestrating various logging functionalities within the application. It handles
* log initialization, middleware registration, log directory management, and
* provides methods for creating log contexts and managing log output levels.
*/
class LogService extends BaseService {
static MODULES = {
path: require('path'),
}
/**
* Defines the modules required by the LogService class.
* This static property contains modules that are used for file path operations.
* @property {Object} MODULES - An object containing required modules.
* @property {Object} MODULES.path - The Node.js path module for handling and resolving file paths.
*/
async _construct () {
this.loggers = [];
this.bufferLogger = null;
}
/**
* Registers a custom logging middleware with the LogService.
* @param {*} callback - The callback function that modifies log parameters before delegation.
*/
register_log_middleware (callback) {
this.loggers[0] = new CustomLogger(this.loggers[0], callback);
}
/**
* Registers logging commands with the command service.
*/
['__on_boot.consolidation'] () {
const commands = this.services.get('commands');
commands.registerCommands('logs', [
@ -332,6 +391,15 @@ class LogService extends BaseService {
}
]);
}
/**
* Registers logging commands with the command service.
*
* This method sets up various logging commands that can be used to
* interact with the log output, such as toggling log display,
* starting/stopping log recording, and toggling log indentation.
*
* @memberof LogService
*/
async _init () {
const config = this.global_config;
@ -423,6 +491,13 @@ class LogService extends BaseService {
globalThis.root_context.set('logger', this.create('root-context'));
}
/**
* Create a new log context with the specified prefix
*
* @param {1} prefix - The prefix for the log context
* @param {*} fields - Optional fields to include in the log context
* @returns {LogContext} A new log context with the specified prefix and fields
*/
create (prefix, fields = {}) {
const logContext = new LogContext(
this,
@ -457,6 +532,15 @@ class LogService extends BaseService {
}
}
/**
* Ensures that a log directory exists for logging purposes.
* This method attempts to create or locate a directory for log files,
* falling back through several predefined paths if the preferred
* directory does not exist or cannot be created.
*
* @throws {Error} If no suitable log directory can be found or created.
*/
ensure_log_directory_ () {
// STEP 1: Try /var/puter/logs/heyputer
{
@ -506,12 +590,24 @@ class LogService extends BaseService {
throw new Error('Unable to create or find log directory');
}
/**
* Generates a sanitized file path for log files.
*
* @param {string} name - The name of the log file, which will be sanitized to remove any path characters.
* @returns {string} A sanitized file path within the log directory.
*/
get_log_file (name) {
// sanitize name: cannot contain path characters
name = name.replace(/[^a-zA-Z0-9-_]/g, '_');
return this.modules.path.join(this.log_directory, name);
}
/**
* Get the most recent log entries from the buffer maintained by the LogService.
* By default, the buffer contains the last 20 log entries.
* @returns
*/
get_log_buffer () {
return this.bufferLogger.buffer;
}

57
src/backend/src/services/runtime-analysis/PagerService.js → src/backend/src/modules/core/PagerService.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@ -17,19 +18,47 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const pdjs = require('@pagerduty/pdjs');
const BaseService = require('../BaseService');
const BaseService = require('../../services/BaseService');
const util = require('util');
const { Context } = require('../../util/context');
/**
* @class PagerService
* @extends BaseService
* @description The PagerService class is responsible for handling pager alerts.
* It extends the BaseService class and provides methods for constructing,
* initializing, and managing alert handlers. The class interacts with PagerDuty
* through the pdjs library to send alerts and integrates with other services via
* command registration.
*/
class PagerService extends BaseService {
static USE = {
Context: 'core.context',
}
async _construct () {
this.config = this.global_config.pager;
this.alertHandlers_ = [];
}
async _init () {
const services = this.services;
/**
* PagerService registers its commands at the consolidation phase because
* the '_init' method of CommandService may not have been called yet.
*/
['__on_boot.consolidation'] () {
this._register_commands(this.services.get('commands'));
}
/**
* Initializes the PagerService instance by setting the configuration and
* initializing an empty alert handler array.
*
* @async
* @memberOf PagerService
* @returns {Promise<void>}
*/
async _init () {
this.alertHandlers_ = [];
if ( ! this.config ) {
@ -37,10 +66,15 @@ class PagerService extends BaseService {
}
this.onInit();
this._register_commands(services.get('commands'));
}
/**
* Initializes PagerDuty configuration and registers alert handlers.
* If PagerDuty is enabled in the configuration, it sets up an alert handler
* to send alerts to PagerDuty.
*
* @method onInit
*/
onInit () {
if ( this.config.pagerduty && this.config.pagerduty.enabled ) {
this.alertHandlers_.push(async alert => {
@ -56,7 +90,7 @@ class PagerService extends BaseService {
server_id: this.global_config.server_id,
};
const ctx = Context.get(undefined, { allow_fallback: true });
const ctx = this.Context.get(undefined, { allow_fallback: true });
// Add request payload if any exists
const req = ctx.get('req');
@ -89,6 +123,15 @@ class PagerService extends BaseService {
}
}
/**
* Sends an alert to all registered alert handlers.
*
* This method iterates through all alert handlers and attempts to send the alert.
* If any handler fails to send the alert, an error message is logged.
*
* @param {Object} alert - The alert object containing details about the alert.
*/
async alert (alert) {
for ( const handler of this.alertHandlers_ ) {
try {

77
src/backend/src/services/ParameterService.js → src/backend/src/modules/core/ParameterService.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"claude"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@ -17,32 +18,37 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const BaseService = require("./BaseService");
const BaseService = require("../../services/BaseService");
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* @class ParameterService
* @extends BaseService
* @description Service class for managing system parameters and their values.
* Provides functionality for creating, getting, setting, and subscribing to parameters.
* Supports parameter binding to instances and includes command registration for parameter management.
* Parameters can have constraints, default values, and change listeners.
*/
class ParameterService extends BaseService {
/**
* Parameter service for managing system-wide parameters
* @extends BaseService
* @class
* @description Handles registration, storage, and access of parameters across services.
* Parameters can be bound to instances, subscribed to for changes, and accessed via commands.
* Each parameter has a unique service-scoped ID and optional constraints.
*/
_construct () {
this.parameters_ = [];
}
_init () {
/**
* Initializes the service by registering commands with the command service.
* This method is called during service startup to set up command handlers
* for parameter management.
* @private
*/
['__on_boot.consolidation'] () {
this._registerCommands(this.services.get('commands'));
}
@ -63,6 +69,13 @@ class ParameterService extends BaseService {
}
}
/**
* Gets the value of a parameter by its ID
* @param {string} id - The unique identifier of the parameter to retrieve
* @returns {Promise<*>} The current value of the parameter
* @throws {Error} If parameter with given ID is not found
*/
async get(id) {
const parameter = this._get_param(id);
return await parameter.get();
@ -87,6 +100,10 @@ class ParameterService extends BaseService {
}
_registerCommands (commands) {
/**
* Registers parameter-related commands with the command service
* @param {Object} commands - The command service instance to register with
*/
const completeParameterName = (args) => {
// The parameter name is the first argument, so return no results if we're on the second or later.
if (args.length > 1)
@ -146,6 +163,13 @@ class ParameterService extends BaseService {
}
}
/**
* @class Parameter
* @description Represents a configurable parameter with value management, constraints, and change notification capabilities.
* Provides functionality for setting/getting values, binding to object instances, and subscribing to value changes.
* Supports validation through configurable constraints and maintains a list of value change listeners.
*/
class Parameter {
constructor(spec) {
this.spec_ = spec;
@ -156,6 +180,14 @@ class Parameter {
}
}
/**
* Sets a new value for the parameter after validating against constraints
* @param {*} value - The new value to set for the parameter
* @throws {Error} If the value fails any constraint checks
* @fires valueListeners with new value and old value
* @async
*/
async set (value) {
for ( const constraint of (this.spec_.constraints ?? []) ) {
if ( ! await constraint.check(value) ) {
@ -170,6 +202,11 @@ class Parameter {
}
}
/**
* Gets the current value of this parameter
* @returns {Promise<*>} The parameter's current value
*/
async get () {
return this.value_;
}

41
src/backend/src/services/runtime-analysis/ProcessEventService.js → src/backend/src/modules/core/ProcessEventService.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"claude"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@ -16,18 +17,36 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { Context } = require("../../util/context");
class ProcessEventService {
constructor ({ services }) {
const BaseService = require("../../services/BaseService");
/**
* Service class that handles process-wide events and errors.
* Provides centralized error handling for uncaught exceptions and unhandled promise rejections.
* Sets up event listeners on the process object to capture and report critical errors
* through the logging and error reporting services.
*
* @class ProcessEventService
*/
class ProcessEventService extends BaseService {
static USE = {
Context: 'core.context',
};
_init () {
const services = this.services;
const log = services.get('log-service').create('process-event-service');
const errors = services.get('error-service').create(log);
// TODO: when the service lifecycle is implemented, but these
// in the init hook
process.on('uncaughtException', async (err, origin) => {
await Context.allow_fallback(async () => {
/**
* Handles uncaught exceptions in the process
* Sets up an event listener that reports errors when uncaught exceptions occur
* @param {Error} err - The uncaught exception error object
* @param {string} origin - The origin of the uncaught exception
* @returns {Promise<void>}
*/
await this.Context.allow_fallback(async () => {
errors.report('process:uncaughtException', {
source: err,
origin,
@ -39,7 +58,13 @@ class ProcessEventService {
});
process.on('unhandledRejection', async (reason, promise) => {
await Context.allow_fallback(async () => {
/**
* Handles unhandled promise rejections by reporting them to the error service
* @param {*} reason - The rejection reason/error
* @param {Promise} promise - The rejected promise
* @returns {Promise<void>} Resolves when error is reported
*/
await this.Context.allow_fallback(async () => {
errors.report('process:unhandledRejection', {
source: reason,
promise,

269
src/backend/src/modules/core/README.md Normal file
View File

@ -0,0 +1,269 @@
# Core2Module
A replacement for CoreModule with as few external relative requires as possible.
This will eventually be the successor to CoreModule, the main module for Puter's backend.
## Services
### AlarmService
AlarmService class is responsible for managing alarms.
It provides methods for creating, clearing, and handling alarms.
#### Listeners
##### `boot.consolidation`
AlarmService registers its commands at the consolidation phase because
the '_init' method of CommandService may not have been called yet.
#### Methods
##### `create`
Method to create an alarm with the given ID, message, and fields.
If the ID already exists, it will be updated with the new fields
and the occurrence count will be incremented.
###### Parameters
- **id:** Unique identifier for the alarm.
- **message:** Message associated with the alarm.
- **fields:** Additional information about the alarm.
##### `clear`
Method to clear an alarm with the given ID.
###### Parameters
- **id:** The ID of the alarm to clear.
##### `get_alarm`
Method to get an alarm by its ID.
###### Parameters
- **id:** The ID of the alarm to get.
### ErrorService
The ErrorService class is responsible for handling and reporting errors within the system.
It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms.
#### Methods
##### `init`
Initializes the ErrorService, setting up the alarm and backup logger services.
##### `create`
Creates an ErrorContext instance with the provided logging context.
###### Parameters
- **log_context:** The logging context to associate with the error reports.
##### `report`
Reports an error with the specified location and details.
The "location" is a string up to the callers discretion to identify
the source of the error.
###### Parameters
- **location:** The location where the error occurred.
- **fields:** The error details to report.
### ExpectationService
#### Listeners
##### `boot.consolidation`
ExpectationService registers its commands at the consolidation phase because
the '_init' method of CommandService may not have been called yet.
#### Methods
##### `expect_eventually`
Registers an expectation to be tracked by the service.
###### Parameters
- **workUnit:** The work unit to track
- **checkpoint:** The checkpoint to expect
### LogService
The `LogService` class extends `BaseService` and is responsible for managing and
orchestrating various logging functionalities within the application. It handles
log initialization, middleware registration, log directory management, and
provides methods for creating log contexts and managing log output levels.
#### Listeners
##### `boot.consolidation`
Registers logging commands with the command service.
#### Methods
##### `register_log_middleware`
Registers a custom logging middleware with the LogService.
###### Parameters
- **callback:** The callback function that modifies log parameters before delegation.
##### `create`
Create a new log context with the specified prefix
###### Parameters
- **prefix:** The prefix for the log context
- **fields:** Optional fields to include in the log context
##### `get_log_file`
Generates a sanitized file path for log files.
###### Parameters
- **name:** The name of the log file, which will be sanitized to remove any path characters.
##### `get_log_buffer`
Get the most recent log entries from the buffer maintained by the LogService.
By default, the buffer contains the last 20 log entries.
### PagerService
#### Listeners
##### `boot.consolidation`
PagerService registers its commands at the consolidation phase because
the '_init' method of CommandService may not have been called yet.
#### Methods
##### `onInit`
Initializes PagerDuty configuration and registers alert handlers.
If PagerDuty is enabled in the configuration, it sets up an alert handler
to send alerts to PagerDuty.
##### `alert`
Sends an alert to all registered alert handlers.
This method iterates through all alert handlers and attempts to send the alert.
If any handler fails to send the alert, an error message is logged.
###### Parameters
- **alert:** The alert object containing details about the alert.
### ProcessEventService
Service class that handles process-wide events and errors.
Provides centralized error handling for uncaught exceptions and unhandled promise rejections.
Sets up event listeners on the process object to capture and report critical errors
through the logging and error reporting services.
## Libraries
### core.expect
### core.util.identutil
#### Functions
##### `randomItem`
Select a random item from an array using a random number generator function.
###### Parameters
- **arr:** The array to select an item from
### core.util.logutil
#### Functions
##### `stringify_log_entry`
Stringifies a log entry into a formatted string for console output.
###### Parameters
- **logEntry:** The log entry object containing:
### stdio
#### Functions
##### `visible_length`
METADATA // {"ai-commented":{"service":"claude"}}
##### `split_lines`
Split a string into lines according to the terminal width,
preserving ANSI escape sequences, and return an array of lines.
###### Parameters
- **str:** The string to split into lines
### core.util.strutil
#### Functions
##### `quot`
METADATA // {"def":"core.util.strutil","ai-params":{"service":"claude"},"ai-commented":{"service":"claude"}}
##### `osclink`
Creates an OSC 8 hyperlink sequence for terminal output
###### Parameters
- **url:** The URL to link to
##### `format_as_usd`
Formats a number as a USD currency string with appropriate decimal places
###### Parameters
- **amount:** The amount to format
## Notes
### Outside Imports
This module has external relative imports. When these are
removed it may become possible to move this module to an
extension.
**Imports:**
- `../../services/BaseService.js`
- `../../util/context.js`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../util/context`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)

97
src/backend/src/services/runtime-analysis/ServerHealthService.js → src/backend/src/modules/core/ServerHealthService.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"xai"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
@ -16,19 +17,45 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const BaseService = require("../BaseService");
const { SECOND } = require("../../util/time");
const { parse_meminfo } = require("../../util/linux");
const { asyncSafeSetInterval, TeePromise } = require("../../util/promise");
const BaseService = require("../../services/BaseService");
const { time, promise } = require("@heyputer/putility").libs;
/**
* The ServerHealthService class provides comprehensive health monitoring for the server.
* It extends the BaseService class to include functionality for:
* - Periodic system checks (e.g., RAM usage, service checks)
* - Managing health check results and failures
* - Triggering alarms for critical conditions
* - Logging and managing statistics for health metrics
*
* This service is designed to work primarily on Linux systems, reading system metrics
* from `/proc/meminfo` and handling alarms via an external 'alarm' service.
*/
class ServerHealthService extends BaseService {
static USE = {
linuxutil: 'core.util.linuxutil'
};
static MODULES = {
fs: require('fs'),
}
/**
* Defines the modules used by ServerHealthService.
* This static property is used to initialize and access system modules required for health checks.
* @type {Object}
* @property {fs} fs - The file system module for reading system information.
*/
_construct () {
this.checks_ = [];
this.failures_ = [];
}
/**
* Initializes the internal checks and failure tracking for the service.
* This method sets up empty arrays to store health checks and their failure statuses.
*
* @private
*/
async _init () {
this.init_service_checks_();
@ -45,7 +72,6 @@ class ServerHealthService extends BaseService {
*/
const min_free_KiB = 1024 * 1024; // 1 GiB
const min_available_KiB = 1024 * 1024 * 2; // 2 GiB
const svc_alarm = this.services.get('alarm');
@ -59,11 +85,19 @@ class ServerHealthService extends BaseService {
if ( this.config.no_system_checks ) return;
/**
* Adds a health check to the service.
*
* @param {string} name - The name of the health check.
* @param {Function} fn - The function to execute for the health check.
* @returns {Object} A chainable object to add failure handlers.
*/
this.add_check('ram-usage', async () => {
const meminfo_text = await this.modules.fs.promises.readFile(
'/proc/meminfo', 'utf8'
);
const meminfo = parse_meminfo(meminfo_text);
const meminfo = this.linuxutil.parse_meminfo(meminfo_text);
const alarm_fields = {
mem_free: meminfo.MemFree,
mem_available: meminfo.MemAvailable,
@ -78,16 +112,43 @@ class ServerHealthService extends BaseService {
});
}
/**
* Initializes service health checks by setting up periodic checks.
* This method configures an interval-based execution of health checks,
* handles timeouts, and manages failure states.
*
* @param {none} - This method does not take any parameters.
* @returns {void} - This method does not return any value.
*/
init_service_checks_ () {
const svc_alarm = this.services.get('alarm');
asyncSafeSetInterval(async () => {
/**
* Initializes periodic health checks for the server.
*
* This method sets up an interval to run all registered health checks
* at a specified frequency. It manages the execution of checks, handles
* timeouts, and logs errors or triggers alarms when checks fail.
*
* @private
* @method init_service_checks_
* @memberof ServerHealthService
* @param {none} - No parameters are passed to this method.
* @returns {void}
*/
promise.asyncSafeSetInterval(async () => {
this.log.tick('service checks');
const check_failures = [];
for ( const { name, fn, chainable } of this.checks_ ) {
const p_timeout = new TeePromise();
const p_timeout = new promise.TeePromise();
/**
* Creates a TeePromise to handle potential timeouts during health checks.
*
* @returns {Promise} A promise that can be resolved or rejected from multiple places.
*/
const timeout = setTimeout(() => {
p_timeout.reject(new Error('Health check timed out'));
}, 5 * SECOND);
}, 5 * time.SECOND);
try {
await Promise.race([
fn(),
@ -120,7 +181,7 @@ class ServerHealthService extends BaseService {
}
this.failures_ = check_failures;
}, 10 * SECOND, null, {
}, 10 * time.SECOND, null, {
onBehindSchedule: (drift) => {
svc_alarm.create(
'health-checks-behind-schedule',
@ -131,6 +192,14 @@ class ServerHealthService extends BaseService {
});
}
/**
* Retrieves the current server health statistics.
*
* @returns {Object} An object containing the current health statistics.
* This method returns a shallow copy of the internal `stats_` object to prevent
* direct manipulation of the service's data.
*/
async get_stats () {
return { ...this.stats_ };
}
@ -147,6 +216,14 @@ class ServerHealthService extends BaseService {
return chainable;
}
/**
* Retrieves the current health status of the server.
*
* @returns {Object} An object containing:
* - `ok` {boolean}: Indicates if all health checks passed.
* - `failed` {Array<string>}: An array of names of failed health checks, if any.
*/
get_status () {
const failures = this.failures_.map(v => v.name);
return {

9
src/backend/src/modules/core/lib/__lib__.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = {
util: {
logutil: require('./log.js'),
identutil: require('./identifier.js'),
stdioutil: require('./stdio.js'),
linuxutil: require('./linux.js'),
},
expect: require('./expect.js'),
};

74
src/backend/src/modules/core/lib/expect.js Normal file
View File

@ -0,0 +1,74 @@
// METADATA // {"def":"core.expect"}
const { v4: uuidv4 } = require('uuid');
/**
* @class WorkUnit
* @description The WorkUnit class represents a unit of work that can be tracked and monitored for checkpoints.
* It includes methods to create instances, set checkpoints, and manage the state of the work unit.
*/
class WorkUnit {
/**
* Represents a unit of work with checkpointing capabilities.
*
* @class
*/
/**
* Creates and returns a new instance of WorkUnit.
*
* @static
* @returns {WorkUnit} A new instance of WorkUnit.
*/
static create () {
return new WorkUnit();
}
/**
* Creates a new instance of the WorkUnit class.
* @static
* @returns {WorkUnit} A new WorkUnit instance.
*/
constructor () {
this.id = uuidv4();
this.checkpoint_ = null;
}
checkpoint (label) {
console.log('CHECKPOINT', label);
this.checkpoint_ = label;
}
}
/**
* @class CheckpointExpectation
* @classdesc The CheckpointExpectation class is used to represent an expectation that a specific checkpoint
* will be reached during the execution of a work unit. It includes methods to check if the checkpoint has
* been reached and to report the results of this check.
*/
class CheckpointExpectation {
constructor (workUnit, checkpoint) {
this.workUnit = workUnit;
this.checkpoint = checkpoint;
}
/**
* Constructor for CheckpointExpectation class.
* Initializes the instance with a WorkUnit and a checkpoint label.
* @param {WorkUnit} workUnit - The work unit associated with the checkpoint.
* @param {string} checkpoint - The checkpoint label to be checked.
*/
check () {
// TODO: should be true if checkpoint was ever reached
return this.workUnit.checkpoint_ == this.checkpoint;
}
report (log) {
if ( this.check() ) return;
log.log(
`operation(${this.workUnit.id}): ` +
`expected ${JSON.stringify(this.checkpoint)} ` +
`and got ${JSON.stringify(this.workUnit.checkpoint_)}.`
);
}
}
module.exports = {
WorkUnit,
CheckpointExpectation,
};

128
src/backend/src/modules/core/lib/identifier.js Normal file
View File

@ -0,0 +1,128 @@
// METADATA // {"def":"core.util.identutil","ai-commented":{"service":"claude"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const adjectives = [
'amazing', 'ambitious', 'articulate', 'cool', 'bubbly', 'mindful', 'noble', 'savvy', 'serene',
'sincere', 'sleek', 'sparkling', 'spectacular', 'splendid', 'spotless', 'stunning',
'awesome', 'beaming', 'bold', 'brilliant', 'cheerful', 'modest', 'motivated',
'friendly', 'fun', 'funny', 'generous', 'gifted', 'graceful', 'grateful',
'passionate', 'patient', 'peaceful', 'perceptive', 'persistent',
'helpful', 'sensible', 'loyal', 'honest', 'clever', 'capable',
'calm', 'smart', 'genius', 'bright', 'charming', 'creative', 'diligent', 'elegant', 'fancy',
'colorful', 'avid', 'active', 'gentle', 'happy', 'intelligent',
'jolly', 'kind', 'lively', 'merry', 'nice', 'optimistic', 'polite',
'quiet', 'relaxed', 'silly', 'witty', 'young',
'strong', 'brave', 'agile', 'bold', 'confident', 'daring',
'fearless', 'heroic', 'mighty', 'powerful', 'valiant', 'wise', 'wonderful', 'zealous',
'warm', 'swift', 'neat', 'tidy', 'nifty', 'lucky', 'keen',
'blue', 'red', 'aqua', 'green', 'orange', 'pink', 'purple', 'cyan', 'magenta', 'lime',
'teal', 'lavender', 'beige', 'maroon', 'navy', 'olive', 'silver', 'gold', 'ivory',
];
const nouns = [
'street', 'roof', 'floor', 'tv', 'idea', 'morning', 'game', 'wheel', 'bag', 'clock', 'pencil', 'pen',
'magnet', 'chair', 'table', 'house', 'room', 'book', 'car', 'tree', 'candle', 'light', 'planet',
'flower', 'bird', 'fish', 'sun', 'moon', 'star', 'cloud', 'rain', 'snow', 'wind', 'mountain',
'river', 'lake', 'sea', 'ocean', 'island', 'bridge', 'road', 'train', 'plane', 'ship', 'bicycle',
'circle', 'square', 'garden', 'harp', 'grass', 'forest', 'rock', 'cake', 'pie', 'cookie', 'candy',
'butterfly', 'computer', 'phone', 'keyboard', 'mouse', 'cup', 'plate', 'glass', 'door',
'window', 'key', 'wallet', 'pillow', 'bed', 'blanket', 'soap', 'towel', 'lamp', 'mirror',
'camera', 'hat', 'shirt', 'pants', 'shoes', 'watch', 'ring',
'necklace', 'ball', 'toy', 'doll', 'kite', 'balloon', 'guitar', 'violin', 'piano', 'drum',
'trumpet', 'flute', 'viola', 'cello', 'harp', 'banjo', 'tuba',
]
const words = {
adjectives,
nouns,
};
/**
* Select a random item from an array using a random number generator function.
*
* @param {Array<T>} arr - The array to select an item from
* @param {function} [random=Math.random] - Random number generator function
* @returns {T} A random item from the array
*/
const randomItem = (arr, random) => arr[Math.floor((random ?? Math.random)() * arr.length)];
/**
* A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999).
* The result is returned as a string with components separated by the specified separator.
* It is useful when you need to create unique identifiers that are also human-friendly.
*
* @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided.
* @param {function} [rng=Math.random] - Random number generator function
* @returns {string} A unique, human-friendly identifier.
*
* @example
*
* let identifier = window.generate_identifier();
* // identifier would be something like 'clever-idea-123'
*
*/
function generate_identifier(separator = '_', rng = Math.random){
// return a random combination of first_adj + noun + number (between 0 and 9999)
// e.g. clever-idea-123
return [
randomItem(adjectives, rng),
randomItem(nouns, rng),
Math.floor(rng() * 10000),
].join(separator);
}
// Character set used for generating human-readable, case-insensitive random codes
const HUMAN_READABLE_CASE_INSENSITIVE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
function generate_random_code(n, {
rng = Math.random,
chars = HUMAN_READABLE_CASE_INSENSITIVE
} = {}) {
let code = '';
for ( let i = 0 ; i < n ; i++ ) {
code += randomItem(chars, rng);
}
return code;
}
/**
* Composes a code by combining a mask string with a base-36 converted number
* @param {string} mask - Initial string template to use as base
* @param {number} value - Number to convert to base-36 and append to the right
* @returns {string} Combined uppercase code
*/
function compose_code(mask, value) {
const right_str = value.toString(36);
let out_str = mask;
console.log('right_str', right_str);
console.log('out_str', out_str);
for ( let i = 0 ; i < right_str.length ; i++ ) {
out_str[out_str.length - 1 - i] = right_str[right_str.length - 1 - i];
}
out_str = out_str.toUpperCase();
return out_str;
}
module.exports = {
randomItem,
generate_identifier,
generate_random_code,
};

7
src/backend/src/util/linux.js → src/backend/src/modules/core/lib/linux.js
View File

@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const SmolUtil = require("./smolutil");
const smol = require('@heyputer/putility').libs.smol;
const parse_meminfo = text => {
const lines = text.split('\n');
@ -26,8 +26,8 @@ const parse_meminfo = text => {
for ( const line of lines ) {
if ( line.trim().length == 0 ) continue;
const [key, value_and_unit] = SmolUtil.split(line, ':', { trim: true });
const [value, _] = SmolUtil.split(value_and_unit, ' ', { trim: true });
const [key, value_and_unit] = smol.split(line, ':', { trim: true });
const [value, _] = smol.split(value_and_unit, ' ', { trim: true });
// note: unit is always 'kB' so we discard it
meminfo[key] = Number.parseInt(value);
}
@ -38,3 +38,4 @@ const parse_meminfo = text => {
module.exports = {
parse_meminfo,
};

73
src/backend/src/modules/core/lib/log.js Normal file
View File

@ -0,0 +1,73 @@
// METADATA // {"def":"core.util.logutil","ai-commented":{"service":"openai-completion","model":"gpt-4o"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const log_epoch = Date.now();
/**
* Stringifies a log entry into a formatted string for console output.
* @param {Object} logEntry - The log entry object containing:
* @param {string} [prefix] - Optional prefix for the log message.
* @param {Object} log_lvl - Log level object with properties for label, escape code, etc.
* @param {string[]} crumbs - Array of context crumbs.
* @param {string} message - The log message.
* @param {Object} fields - Additional fields to be included in the log.
* @param {Object} objects - Objects to be logged.
* @returns {string} A formatted string representation of the log entry.
*/
const stringify_log_entry = ({ prefix, log_lvl, crumbs, message, fields, objects }) => {
const { colorize } = require('json-colorizer');
let lines = [], m;
const lf = () => {
if ( ! m ) return;
lines.push(m);
m = '';
}
m = prefix ? `${prefix} ` : '';
m += `\x1B[${log_lvl.esc}m[${log_lvl.label}\x1B[0m`;
for ( const crumb of crumbs ) {
m += `::${crumb}`;
}
m += `\x1B[${log_lvl.esc}m]\x1B[0m`;
if ( fields.timestamp ) {
// display seconds since logger epoch
const n = (fields.timestamp - log_epoch) / 1000;
m += ` (${n.toFixed(3)}s)`;
}
m += ` ${message} `;
lf();
for ( const k in fields ) {
if ( k === 'timestamp' ) continue;
let v; try {
v = colorize(JSON.stringify(fields[k]));
} catch (e) {
v = '' + fields[k];
}
m += ` \x1B[1m${k}:\x1B[0m ${v}`;
lf();
}
return lines.join('\n');
};
module.exports = {
stringify_log_entry,
log_epoch,
};

70
src/backend/src/modules/core/lib/stdio.js Normal file
View File

@ -0,0 +1,70 @@
// METADATA // {"ai-commented":{"service":"claude"}}
/*
* Copyright (C) 2024 Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* Strip ANSI escape sequences from a string (e.g. color codes)
* and then return the length of the resulting string.
*
* @param {string} str - The string to calculate visible length for
* @returns {number} The length of the string without ANSI escape sequences
*/
const visible_length = (str) => {
// eslint-disable-next-line no-control-regex
return str.replace(/\x1b\[[0-9;]*m/g, '').length;
};
/**
* Split a string into lines according to the terminal width,
* preserving ANSI escape sequences, and return an array of lines.
*
* @param {string} str The string to split into lines
* @returns {string[]} Array of lines split according to terminal width
*/
const split_lines = (str) => {
const lines = [];
let line = '';
let line_length = 0;
for (const c of str) {
line += c;
if (c === '\n') {
lines.push(line);
line = '';
line_length = 0;
} else {
line_length++;
if (line_length >= process.stdout.columns) {
lines.push(line);
line = '';
line_length = 0;
}
}
}
if (line.length) {
lines.push(line);
}
return lines;
};
module.exports = {
visible_length,
split_lines,
};

20
src/backend/src/modules/perfmon/PerfMonModule.js Normal file
View File

@ -0,0 +1,20 @@
const { AdvancedBase } = require("@heyputer/putility");
/**
* Enable this module when you want performance monitoring.
*
* Performance monitoring requires additional setup. Jaegar should be installed
* and running.
*/
class PerfMonModule extends AdvancedBase {
async install (context) {
const services = context.get('services');
const TelemetryService = require("./TelemetryService");
services.registerService('telemetry', TelemetryService);
}
}
module.exports = {
PerfMonModule,
};

48
src/backend/src/monitor/Telemetry.js → src/backend/src/modules/perfmon/TelemetryService.js
View File

@ -16,6 +16,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const opentelemetry = require("@opentelemetry/api");
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { PeriodicExportingMetricReader, ConsoleMetricExporter } = require('@opentelemetry/sdk-metrics');
@ -25,17 +26,13 @@ const { SemanticResourceAttributes } = require("@opentelemetry/semantic-conventi
const { NodeTracerProvider } = require("@opentelemetry/sdk-trace-node");
const { ConsoleSpanExporter, BatchSpanProcessor } = require("@opentelemetry/sdk-trace-base");
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');
const config = require('../config');
const config = require('../../config');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-grpc');
class TelemetryService {
static instance_ = null;
static getInstance () {
if ( this.instance_ ) return this.instance_;
return this.instance_ = new TelemetryService();
}
const BaseService = require('../../services/BaseService');
constructor () {
class TelemetryService extends BaseService {
_construct () {
const resource = Resource.default().merge(
new Resource({
[SemanticResourceAttributes.SERVICE_NAME]: "puter-backend",
@ -61,6 +58,33 @@ class TelemetryService {
});
this.sdk = sdk;
this.sdk.start();
this.tracer_ = opentelemetry.trace.getTracer(
'puter-tracer'
);
}
_init () {
const svc_context = this.services.get('context');
svc_context.register_context_hook('pre_arun', ({ hints, trace_name, callback, replace_callback }) => {
if ( ! trace_name ) return;
if ( ! hints.trace ) return;
console.log('APPLYING TRACE NAME', trace_name);
replace_callback(async () => {
return await this.tracer_.startActiveSpan(trace_name, async span => {
try {
return await callback();
} catch (error) {
span.setStatus({ code: opentelemetry.SpanStatusCode.ERROR, message: error.message });
throw error;
} finally {
span.end();
}
});
});
});
}
getConfiguredExporter_() {
@ -69,12 +93,6 @@ class TelemetryService {
}
const exporter = new ConsoleSpanExporter();
}
start () {
// this.sdk.start();
}
}
module.exports = {
TelemetryService
}
module.exports = TelemetryService;

206
src/backend/src/modules/puterai/AIChatService.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const APIError = require("../../api/APIError");
const { PermissionUtil } = require("../../services/auth/PermissionService");
const BaseService = require("../../services/BaseService");
@ -5,15 +6,33 @@ const { DB_WRITE } = require("../../services/database/consts");
const { TypeSpec } = require("../../services/drivers/meta/Construct");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { Context } = require("../../util/context");
const { AsModeration } = require("./lib/AsModeration");
// Maximum number of fallback attempts when a model fails, including the first attempt
const MAX_FALLBACKS = 3 + 1; // includes first attempt
/**
* AIChatService class extends BaseService to provide AI chat completion functionality.
* Manages multiple AI providers, models, and fallback mechanisms for chat interactions.
* Handles model registration, usage tracking, cost calculation, content moderation,
* and implements the puter-chat-completion driver interface. Supports streaming responses
* and maintains detailed model information including pricing and capabilities.
*/
class AIChatService extends BaseService {
static MODULES = {
kv: globalThis.kv,
uuidv4: require('uuid').v4,
}
/**
* Initializes the service by setting up core properties.
* Creates empty arrays for providers and model lists,
* and initializes an empty object for the model map.
* Called during service instantiation.
* @private
*/
_construct () {
this.providers = [];
@ -21,6 +40,13 @@ class AIChatService extends BaseService {
this.detail_model_list = [];
this.detail_model_map = {};
}
/**
* Initializes the service by setting up empty arrays and maps for providers and models.
* This method is called during service construction to establish the initial state.
* Creates empty arrays for providers, simple model list, and detailed model list,
* as well as an empty object for the detailed model map.
* @private
*/
_init () {
this.kvkey = this.modules.uuidv4();
@ -28,6 +54,8 @@ class AIChatService extends BaseService {
const svc_event = this.services.get('event');
svc_event.on('ai.prompt.report-usage', async (_, details) => {
if ( details.service_used === 'fake-chat' ) return;
const values = {
user_id: details.actor?.type?.user?.id,
app_id: details.actor?.type?.app?.id ?? null,
@ -65,8 +93,32 @@ class AIChatService extends BaseService {
await this.db.insert('ai_usage', values);
});
const svc_apiErrpr = this.services.get('api-error');
svc_apiErrpr.register({
max_tokens_exceeded: {
status: 400,
message: ({ input_tokens, max_tokens }) =>
`Input exceeds maximum token count. ` +
`Input has ${input_tokens} tokens, ` +
`but the maximum is ${max_tokens}.`,
},
});
}
/**
* Handles consolidation during service boot by registering service aliases
* and populating model lists/maps from providers.
*
* Registers each provider as an 'ai-chat' service alias and fetches their
* available models and pricing information. Populates:
* - simple_model_list: Basic list of supported models
* - detail_model_list: Detailed model info including costs
* - detail_model_map: Maps model IDs/aliases to their details
*
* @returns {Promise<void>}
*/
async ['__on_boot.consolidation'] () {
{
const svc_driver = this.services.get('driver')
@ -76,13 +128,21 @@ class AIChatService extends BaseService {
}
}
// TODO: get models and pricing for each model
for ( const provider of this.providers ) {
const delegate = this.services.get(provider.service_name)
.as('puter-chat-completion');
// Populate simple model list
{
/**
* Populates the simple model list by fetching available models from the delegate service.
* Wraps the delegate.list() call in a try-catch block to handle potential errors gracefully.
* If the call fails, logs the error and returns an empty array to avoid breaking the service.
* The fetched models are added to this.simple_model_list.
*
* @private
* @returns {Promise<void>}
*/
const models = await (async () => {
try {
return await delegate.list() ?? [];
@ -96,6 +156,14 @@ class AIChatService extends BaseService {
// Populate detail model list and map
{
/**
* Populates the detail model list and map with model information from the provider.
* Fetches detailed model data including pricing and capabilities.
* Handles model aliases and potential conflicts by storing multiple models in arrays.
* Annotates models with their provider service name.
* Catches and logs any errors during model fetching.
* @private
*/
const models = await (async () => {
try {
return await delegate.models() ?? [];
@ -112,6 +180,13 @@ class AIChatService extends BaseService {
});
}
this.detail_model_list.push(...annotated_models);
/**
* Helper function to set or push a model into the detail_model_map.
* If there's no existing entry for the key, sets it directly.
* If there's a conflict, converts the entry to an array and pushes the new model.
* @param {string} key - The model ID or alias
* @param {Object} model - The model details to add
*/
const set_or_push = (key, model) => {
// Typical case: no conflict
if ( ! this.detail_model_map[key] ) {
@ -153,16 +228,46 @@ class AIChatService extends BaseService {
}
},
['puter-chat-completion']: {
/**
* Implements the 'puter-chat-completion' interface methods for AI chat functionality.
* Handles model selection, fallbacks, usage tracking, and moderation.
* Contains methods for listing available models, completing chat prompts,
* and managing provider interactions.
*
* @property {Object} models - Available AI models with details like costs
* @property {Object} list - Simplified list of available models
* @property {Object} complete - Main method for chat completion requests
* @param {Object} parameters - Chat completion parameters including model and messages
* @returns {Promise<Object>} Chat completion response with usage stats
* @throws {Error} If service is called directly or no fallback models available
*/
async models () {
const delegate = this.get_delegate();
if ( ! delegate ) return await this.models_();
return await delegate.models();
},
/**
* Returns list of available AI models with detailed information
*
* Delegates to the intended service's models() method if a delegate exists,
* otherwise returns the internal detail_model_list containing all available models
* across providers with their capabilities and pricing information.
*
* @returns {Promise<Array>} Array of model objects with details like id, provider, cost, etc.
*/
async list () {
const delegate = this.get_delegate();
if ( ! delegate ) return await this.list_();
return await delegate.list();
},
/**
* Lists available AI models in a simplified format
*
* Returns a list of basic model information from all registered providers.
* This is a simpler version compared to models() that returns less detailed info.
*
* @returns {Promise<Array>} Array of simplified model objects
*/
async complete (parameters) {
const client_driver_call = Context.get('client_driver_call');
let { test_mode, intended_service, response_metadata } = client_driver_call;
@ -174,6 +279,7 @@ class AIChatService extends BaseService {
intended_service,
parameters
};
await svc_event.emit('ai.prompt.validate', event);
if ( ! event.allow ) {
test_mode = true;
}
@ -188,6 +294,9 @@ class AIChatService extends BaseService {
if ( test_mode ) {
intended_service = 'fake-chat';
if ( event.abuse ) {
parameters.model = 'abuse';
}
}
if ( intended_service === this.service_name ) {
@ -195,9 +304,11 @@ class AIChatService extends BaseService {
}
const svc_driver = this.services.get('driver');
let ret, error, errors = [];
let ret, error;
let service_used = intended_service;
let model_used = this.get_model_from_request(parameters);
let model_used = this.get_model_from_request(parameters, {
intended_service
});
await this.check_usage_({
actor: Context.get('actor'),
service: service_used,
@ -219,7 +330,6 @@ class AIChatService extends BaseService {
tried.push(model);
error = e;
errors.push(e);
console.error(e);
this.log.error('error calling service', {
intended_service,
@ -272,7 +382,6 @@ class AIChatService extends BaseService {
};
} catch (e) {
error = e;
errors.push(e);
tried.push(fallback_model_name);
this.log.error('error calling fallback', {
intended_service,
@ -330,6 +439,17 @@ class AIChatService extends BaseService {
}
}
/**
* Checks if the user has permission to use AI services and verifies usage limits
*
* @param {Object} params - The check parameters
* @param {Object} params.actor - The user/actor making the request
* @param {string} params.service - The AI service being used
* @param {string} params.model - The model being accessed
* @throws {APIError} If usage is not allowed or limits are exceeded
* @private
*/
async check_usage_ ({ actor, service, model }) {
const svc_permission = this.services.get('permission');
const svc_event = this.services.get('event');
@ -359,12 +479,21 @@ class AIChatService extends BaseService {
}
}
/**
* Moderates chat messages for inappropriate content using OpenAI's moderation service
*
* @param {Object} params - The parameters object
* @param {Array} params.messages - Array of chat messages to moderate
* @returns {Promise<boolean>} Returns true if content is appropriate, false if flagged
*
* @description
* Extracts text content from messages and checks each against OpenAI's moderation.
* Handles both string content and structured message objects.
* Returns false immediately if any message is flagged as inappropriate.
* Returns true if OpenAI service is unavailable or all messages pass moderation.
*/
async moderate ({ messages }) {
const svc_openai = this.services.get('openai-completion');
// We can't use moderation of openai service isn't available
if ( ! svc_openai ) return true;
for ( const msg of messages ) {
const texts = [];
if ( typeof msg.content === 'string' ) texts.push(msg.content);
@ -379,20 +508,67 @@ class AIChatService extends BaseService {
const fulltext = texts.join('\n');
const mod_result = await svc_openai.check_moderation(fulltext);
let mod_last_error = null;
let mod_result = null;
try {
const svc_openai = this.services.get('openai-completion');
mod_result = await svc_openai.check_moderation(fulltext);
if ( mod_result.flagged ) return false;
continue;
} catch (e) {
console.error(e);
mod_last_error = e;
}
try {
const svc_claude = this.services.get('claude');
const chat = svc_claude.as('puter-chat-completion');
const mod = new AsModeration({
chat,
model: 'claude-3-haiku-20240307',
})
if ( ! await mod.moderate(fulltext) ) {
return false;
}
mod_last_error = null;
continue;
} catch (e) {
console.error(e);
mod_last_error = e;
}
if ( mod_last_error ) {
this.log.error('moderation error', {
fulltext,
mod_last_error,
});
throw new Error('no working moderation service');
}
}
return true;
}
async models_ () {
return this.detail_model_list;
}
/**
* Returns a list of available AI models with basic details
* @returns {Promise<Array>} Array of simple model objects containing basic model information
*/
async list_ () {
return this.simple_model_list;
}
/**
* Gets the appropriate delegate service for handling chat completion requests.
* If the intended service is this service (ai-chat), returns undefined.
* Otherwise returns the intended service wrapped as a puter-chat-completion interface.
*
* @returns {Object|undefined} The delegate service or undefined if intended service is ai-chat
*/
get_delegate () {
const client_driver_call = Context.get('client_driver_call');
if ( client_driver_call.intended_service === this.service_name ) {
@ -431,7 +607,7 @@ class AIChatService extends BaseService {
// Calculate the sorted list
const models = this.detail_model_list;
sorted_models = models.sort((a, b) => {
sorted_models = models.toSorted((a, b) => {
return Math.sqrt(
Math.pow(a.cost.input - target_model.cost.input, 2) +
Math.pow(a.cost.output - target_model.cost.output, 2)
@ -463,10 +639,14 @@ class AIChatService extends BaseService {
});
}
get_model_from_request (parameters) {
get_model_from_request (parameters, modified_context = {}) {
const client_driver_call = Context.get('client_driver_call');
let { intended_service } = client_driver_call;
if ( modified_context.intended_service ) {
intended_service = modified_context.intended_service;
}
let model = parameters.model;
if ( ! model ) {
const service = this.services.get(intended_service);

14
src/backend/src/modules/puterai/AIInterfaceService.js
View File

@ -1,6 +1,20 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const BaseService = require("../../services/BaseService");
/**
* Service class that manages AI interface registrations and configurations.
* Handles registration of various AI services including OCR, chat completion,
* image generation, and text-to-speech interfaces. Each interface defines
* its available methods, parameters, and expected results.
* @extends BaseService
*/
class AIInterfaceService extends BaseService {
/**
* Service class for managing AI interface registrations and configurations.
* Extends the base service to provide AI-related interface management.
* Handles registration of OCR, chat completion, image generation, and TTS interfaces.
*/
async ['__on_driver.register.interfaces'] () {
const svc_registry = this.services.get('registry');
const col_interfaces = svc_registry.get('interfaces');

12
src/backend/src/modules/puterai/AITestModeService.js
View File

@ -1,6 +1,18 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const BaseService = require("../../services/BaseService");
/**
* Service class that handles AI test mode functionality.
* Extends BaseService to register test services for AI chat completions.
* Used for testing and development of AI-related features by providing
* a mock implementation of the chat completion service.
*/
class AITestModeService extends BaseService {
/**
* Service for managing AI test mode functionality
* @extends BaseService
*/
async _init () {
const svc_driver = this.services.get('driver');
svc_driver.register_test_service('puter-chat-completion', 'ai-chat');

55
src/backend/src/modules/puterai/AWSPollyService.js
View File

@ -1,12 +1,28 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PollyClient, SynthesizeSpeechCommand, DescribeVoicesCommand } = require("@aws-sdk/client-polly");
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
/**
* AWSPollyService class provides text-to-speech functionality using Amazon Polly.
* Extends BaseService to integrate with AWS Polly for voice synthesis operations.
* Implements voice listing, speech synthesis, and voice selection based on language.
* Includes caching for voice descriptions and supports both text and SSML inputs.
* @extends BaseService
*/
class AWSPollyService extends BaseService {
static MODULES = {
kv: globalThis.kv,
}
/**
* Initializes the service by creating an empty clients object.
* This method is called during service construction to set up
* the internal state needed for AWS Polly client management.
* @returns {Promise<void>}
*/
async _construct () {
this.clients_ = {};
}
@ -18,6 +34,14 @@ class AWSPollyService extends BaseService {
}
},
['puter-tts']: {
/**
* Implements the driver interface methods for text-to-speech functionality
* Contains methods for listing available voices and synthesizing speech
* @interface
* @property {Object} list_voices - Lists available Polly voices with language info
* @property {Object} synthesize - Converts text to speech using specified voice/language
* @property {Function} supports_test_mode - Indicates test mode support for methods
*/
async list_voices () {
const polly_voices = await this.describe_voices();
@ -64,6 +88,12 @@ class AWSPollyService extends BaseService {
}
}
/**
* Creates AWS credentials object for authentication
* @private
* @returns {Object} Object containing AWS access key ID and secret access key
*/
_create_aws_credentials () {
return {
accessKeyId: this.config.aws.access_key,
@ -86,6 +116,13 @@ class AWSPollyService extends BaseService {
return this.clients_[region];
}
/**
* Describes available AWS Polly voices and caches the results
* @returns {Promise<Object>} Response containing array of voice details in Voices property
* @description Fetches voice information from AWS Polly API and caches it for 10 minutes
* Uses KV store for caching to avoid repeated API calls
*/
async describe_voices () {
let voices = this.modules.kv.get('svc:polly:voices');
if ( voices ) {
@ -109,6 +146,17 @@ class AWSPollyService extends BaseService {
return response;
}
/**
* Synthesizes speech from text using AWS Polly
* @param {string} text - The text to synthesize
* @param {Object} options - Synthesis options
* @param {string} options.format - Output audio format (e.g. 'mp3')
* @param {string} [options.voice_id] - AWS Polly voice ID to use
* @param {string} [options.language] - Language code (e.g. 'en-US')
* @param {string} [options.text_type] - Type of input text ('text' or 'ssml')
* @returns {Promise<AWS.Polly.SynthesizeSpeechOutput>} The synthesized speech response
*/
async synthesize_speech (text, { format, voice_id, language, text_type }) {
const client = this._get_client(this.config.aws.region);
@ -140,6 +188,13 @@ class AWSPollyService extends BaseService {
return response;
}
/**
* Attempts to find an appropriate voice for the given language code
* @param {string} language - The language code to find a voice for (e.g. 'en-US')
* @returns {Promise<?string>} The voice ID if found, null if no matching voice exists
* @private
*/
async maybe_get_language_appropriate_voice_ (language) {
const voices = await this.describe_voices();

49
src/backend/src/modules/puterai/AWSTextractService.js
View File

@ -1,9 +1,23 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { TextractClient, AnalyzeDocumentCommand, InvalidS3ObjectException } = require("@aws-sdk/client-textract");
const BaseService = require("../../services/BaseService");
const APIError = require("../../api/APIError");
/**
* AWSTextractService class - Provides OCR (Optical Character Recognition) functionality using AWS Textract
* Extends BaseService to integrate with AWS Textract for document analysis and text extraction.
* Implements driver capabilities and puter-ocr interface for document recognition.
* Handles both S3-stored and buffer-based document processing with automatic region management.
*/
class AWSTextractService extends BaseService {
/**
* AWS Textract service for OCR functionality
* Provides document analysis capabilities using AWS Textract API
* Implements interfaces for OCR recognition and driver capabilities
* @extends BaseService
*/
_construct () {
this.clients_ = {};
}
@ -15,6 +29,13 @@ class AWSTextractService extends BaseService {
}
},
['puter-ocr']: {
/**
* Performs OCR recognition on a document using AWS Textract
* @param {Object} params - Recognition parameters
* @param {Object} params.source - The document source to analyze
* @param {boolean} params.test_mode - If true, returns sample test output instead of processing
* @returns {Promise<Object>} Recognition results containing blocks of text with confidence scores
*/
async recognize ({ source, test_mode }) {
if ( test_mode ) {
return {
@ -61,6 +82,12 @@ class AWSTextractService extends BaseService {
},
};
/**
* Creates AWS credentials object for authentication
* @private
* @returns {Object} Object containing AWS access key ID and secret access key
*/
_create_aws_credentials () {
return {
accessKeyId: this.config.aws.access_key,
@ -83,6 +110,15 @@ class AWSTextractService extends BaseService {
return this.clients_[region];
}
/**
* Analyzes a document using AWS Textract to extract text and layout information
* @param {FileFacade} file_facade - Interface to access the document file
* @returns {Promise<Object>} The raw Textract API response containing extracted text blocks
* @throws {Error} If document analysis fails or no suitable input format is available
* @description Processes document through Textract's AnalyzeDocument API with LAYOUT feature.
* Will attempt to use S3 direct access first, falling back to buffer upload if needed.
*/
async analyze_document (file_facade) {
const {
client, document, using_s3
@ -119,6 +155,18 @@ class AWSTextractService extends BaseService {
throw new Error('expected to be unreachable');
}
/**
* Gets AWS client and document configuration for Textract processing
* @param {Object} file_facade - File facade object containing document source info
* @param {boolean} [force_buffer] - If true, forces using buffer instead of S3
* @returns {Promise<Object>} Object containing:
* - client: Configured AWS Textract client
* - document: Document configuration for Textract
* - using_s3: Boolean indicating if using S3 source
* @throws {APIError} If file does not exist
* @throws {Error} If no suitable input format is available
*/
async _get_client_and_document (file_facade, force_buffer) {
const try_s3info = await file_facade.get('s3-info');
if ( try_s3info && ! force_buffer ) {
@ -137,7 +185,6 @@ class AWSTextractService extends BaseService {
const try_buffer = await file_facade.get('buffer');
if ( try_buffer ) {
const base64 = try_buffer.toString('base64');
return {
client: this._get_client(),
document: {

14
src/backend/src/modules/puterai/ClaudeEnoughService.js
View File

@ -1,3 +1,4 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { XAIService } = require("./XAIService");
const CLAUDE_ENOUGH_PROMPT = `
@ -19,7 +20,20 @@ const CLAUDE_ENOUGH_PROMPT = `
user of the driver interface (typically an app on Puter):
`.replace('\n', ' ').trim();
/**
* ClaudeEnoughService - A service class that implements a Claude-like AI interface
* Extends XAIService to provide Claude-compatible responses while using alternative AI models.
* Includes custom system prompts and model adaptation to simulate Claude's behavior
* in the Puter platform's chat completion interface.
*/
class ClaudeEnoughService extends XAIService {
/**
* Service that emulates Claude's behavior using alternative AI models
* @extends XAIService
* @description Provides a Claude-like interface while using other AI models as the backend.
* Includes custom system prompts and model adaptations to approximate Claude's behavior.
*/
get_system_prompt () {
return CLAUDE_ENOUGH_PROMPT;
}

80
src/backend/src/modules/puterai/ClaudeService.js
View File

@ -1,10 +1,11 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { default: Anthropic } = require("@anthropic-ai/sdk");
const BaseService = require("../../services/BaseService");
const { whatis } = require("../../util/langutil");
const { PassThrough } = require("stream");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const APIError = require("../../api/APIError");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
const PUTER_PROMPT = `
You are running on an open-source platform called Puter,
@ -15,13 +16,27 @@ const PUTER_PROMPT = `
user of the driver interface (typically an app on Puter):
`.replace('\n', ' ').trim();
const MAX_CLAUDE_INPUT_TOKENS = 10000;
/**
* ClaudeService class extends BaseService to provide integration with Anthropic's Claude AI models.
* Implements the puter-chat-completion interface for handling AI chat interactions.
* Manages message streaming, token limits, model selection, and API communication with Claude.
* Supports system prompts, message adaptation, and usage tracking.
* @extends BaseService
*/
class ClaudeService extends BaseService {
static MODULES = {
Anthropic: require('@anthropic-ai/sdk'),
}
/**
* Initializes the Claude service by creating an Anthropic client instance
* and registering this service as a provider with the AI chat service.
* @private
* @returns {Promise<void>}
*/
async _init () {
this.anthropic = new Anthropic({
apiKey: this.config.apiKey
@ -34,15 +49,34 @@ class ClaudeService extends BaseService {
});
}
/**
* Returns the default model identifier for Claude API interactions
* @returns {string} The default model ID 'claude-3-5-sonnet-latest'
*/
get_default_model () {
return 'claude-3-5-sonnet-latest';
}
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implements the puter-chat-completion interface for Claude AI models
* @param {Object} options - Configuration options for the chat completion
* @param {Array} options.messages - Array of message objects containing the conversation history
* @param {boolean} options.stream - Whether to stream the response
* @param {string} [options.model] - The Claude model to use, defaults to claude-3-5-sonnet-latest
* @returns {TypedValue|Object} Returns either a TypedValue with streaming response or a completion object
*/
async models () {
return await this.models_();
},
/**
* Returns a list of available model names including their aliases
* @returns {Promise<string[]>} Array of model identifiers and their aliases
* @description Retrieves all available Claude model IDs and their aliases,
* flattening them into a single array of strings that can be used for model selection
*/
async list () {
const models = await this.models_();
const model_names = [];
@ -54,6 +88,14 @@ class ClaudeService extends BaseService {
}
return model_names;
},
/**
* Completes a chat interaction with the Claude AI model
* @param {Object} options - The completion options
* @param {Array} options.messages - Array of chat messages to process
* @param {boolean} options.stream - Whether to stream the response
* @param {string} [options.model] - The Claude model to use, defaults to service default
* @returns {TypedValue|Object} Returns either a TypedValue with streaming response or a completion object
*/
async complete ({ messages, stream, model }) {
const adapted_messages = [];
@ -84,24 +126,11 @@ class ClaudeService extends BaseService {
adapted_messages.push(message);
if ( message.role === 'user' ) {
previous_was_user = true;
} else {
previous_was_user = false;
}
}
const token_count = (() => {
const text = JSON.stringify(adapted_messages) +
JSON.stringify(system_prompts);
// This is the most accurate token counter available for Claude.
return text.length / 4;
})();
if ( token_count > MAX_CLAUDE_INPUT_TOKENS ) {
throw APIError.create('max_tokens_exceeded', null, {
input_tokens: token_count,
max_tokens: MAX_CLAUDE_INPUT_TOKENS,
});
}
if ( stream ) {
let usage_promise = new TeePromise();
@ -114,7 +143,7 @@ class ClaudeService extends BaseService {
(async () => {
const completion = await this.anthropic.messages.stream({
model: model ?? this.get_default_model(),
max_tokens: 1000,
max_tokens: (model === 'claude-3-5-sonnet-20241022' || model === 'claude-3-5-sonnet-20240620') ? 8192 : 4096,
temperature: 0,
system: PUTER_PROMPT + JSON.stringify(system_prompts),
messages: adapted_messages,
@ -151,7 +180,7 @@ class ClaudeService extends BaseService {
const msg = await this.anthropic.messages.create({
model: model ?? this.get_default_model(),
max_tokens: 1000,
max_tokens: (model === 'claude-3-5-sonnet-20241022' || model === 'claude-3-5-sonnet-20240620') ? 8192 : 4096,
temperature: 0,
system: PUTER_PROMPT + JSON.stringify(system_prompts),
messages: adapted_messages,
@ -165,6 +194,19 @@ class ClaudeService extends BaseService {
}
}
/**
* Retrieves available Claude AI models and their specifications
* @returns {Promise<Array>} Array of model objects containing:
* - id: Model identifier
* - name: Display name
* - aliases: Alternative names for the model
* - context: Maximum context window size
* - cost: Pricing details (currency, token counts, input/output costs)
* - qualitative_speed: Relative speed rating
* - max_output: Maximum output tokens
* - training_cutoff: Training data cutoff date
*/
async models_ () {
return [
{

38
src/backend/src/modules/puterai/FakeChatService.js
View File

@ -1,11 +1,37 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { default: dedent } = require("dedent");
const BaseService = require("../../services/BaseService");
/**
* FakeChatService - A mock implementation of a chat service that extends BaseService.
* Provides fake chat completion responses using Lorem Ipsum text generation.
* Used for testing and development purposes when a real chat service is not needed.
* Implements the 'puter-chat-completion' interface with list() and complete() methods.
*/
class FakeChatService extends BaseService {
get_default_model () {
return 'fake';
}
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implementation interface for the puter-chat-completion service.
* Provides fake chat completion functionality for testing purposes.
* Contains methods for listing available models and generating mock responses.
* @interface
*/
async list () {
return ['fake'];
},
/**
* Simulates a chat completion request by generating random Lorem Ipsum text
* @param {Object} params - The completion parameters
* @param {Array} params.messages - Array of chat messages (unused in fake implementation)
* @param {boolean} params.stream - Whether to stream the response (unused in fake implementation)
* @param {string} params.model - The model to use (unused in fake implementation)
* @returns {Object} A simulated chat completion response with Lorem Ipsum content
*/
async complete ({ messages, stream, model }) {
const { LoremIpsum } = require('lorem-ipsum');
const li = new LoremIpsum({
@ -28,7 +54,13 @@ class FakeChatService extends BaseService {
"content": [
{
"type": "text",
"text": li.generateParagraphs(
"text": model === 'abuse' ? dedent(`
This is a message from ${
this.global_config.origin}. We have detected abuse of our services.
If you are seeing this on another website, please report it to ${
this.global_config.abuse_email ?? 'hi@puter.com'}
`) : li.generateParagraphs(
Math.floor(Math.random() * 3) + 1
)
}
@ -40,6 +72,10 @@ class FakeChatService extends BaseService {
"output_tokens": 1
}
},
"usage": {
"input_tokens": 0,
"output_tokens": 1
},
"logprobs": null,
"finish_reason": "stop"
}

58
src/backend/src/modules/puterai/GroqAIService.js
View File

@ -1,14 +1,31 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PassThrough } = require("stream");
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { nou } = require("../../util/langutil");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
/**
* Service class for integrating with Groq AI's language models.
* Extends BaseService to provide chat completion capabilities through the Groq API.
* Implements the puter-chat-completion interface for model management and text generation.
* Supports both streaming and non-streaming responses, handles multiple models including
* various versions of Llama, Mixtral, and Gemma, and manages usage tracking.
* @class GroqAIService
* @extends BaseService
*/
class GroqAIService extends BaseService {
static MODULES = {
Groq: require('groq-sdk'),
}
/**
* Initializes the GroqAI service by setting up the Groq client and registering with the AI chat provider
* @returns {Promise<void>}
* @private
*/
async _init () {
const Groq = require('groq-sdk');
this.client = new Groq({
@ -22,20 +39,47 @@ class GroqAIService extends BaseService {
});
}
/**
* Returns the default model ID for the Groq AI service
* @returns {string} The default model ID 'llama-3.1-8b-instant'
*/
get_default_model () {
return 'llama-3.1-8b-instant';
}
static IMPLEMENTS = {
'puter-chat-completion': {
/**
* Defines the interface implementations for the puter-chat-completion service
* Contains methods for listing models and handling chat completions
* @property {Object} models - Returns available AI models
* @property {Object} list - Lists raw model data from the Groq API
* @property {Object} complete - Handles chat completion requests with optional streaming
* @returns {Object} Interface implementation object
*/
async models () {
return await this.models_();
},
/**
* Lists available AI models from the Groq API
* @returns {Promise<Array>} Array of model objects from the API's data field
* @description Unwraps and returns the model list from the Groq API response,
* which comes wrapped in an object with {object: "list", data: [...]}
*/
async list () {
// They send: { "object": "list", data }
const funny_wrapper = await this.client.models.list();
return funny_wrapper.data;
},
/**
* Completes a chat interaction using the Groq API
* @param {Object} options - The completion options
* @param {Array<Object>} options.messages - Array of message objects containing the conversation history
* @param {string} [options.model] - The model ID to use for completion. Defaults to service's default model
* @param {boolean} [options.stream] - Whether to stream the response
* @returns {TypedValue|Object} Returns either a TypedValue with streaming response or completion object with usage stats
*/
async complete ({ messages, model, stream }) {
for ( let i = 0; i < messages.length; i++ ) {
const message = messages[i];
@ -101,6 +145,18 @@ class GroqAIService extends BaseService {
}
};
/**
* Returns an array of available AI models with their specifications
*
* Each model object contains:
* - id: Unique identifier for the model
* - name: Human-readable name
* - context: Maximum context window size in tokens
* - cost: Pricing details including currency and token rates
*
* @returns {Array<Object>} Array of model specification objects
*/
models_ () {
return [
{

54
src/backend/src/modules/puterai/MistralAIService.js
View File

@ -1,15 +1,30 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PassThrough } = require("stream");
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { nou } = require("../../util/langutil");
const axios = require('axios');
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
/**
* MistralAIService class extends BaseService to provide integration with the Mistral AI API.
* Implements chat completion functionality with support for various Mistral models including
* mistral-large, pixtral, codestral, and ministral variants. Handles both streaming and
* non-streaming responses, token usage tracking, and model management. Provides cost information
* for different models and implements the puter-chat-completion interface.
*/
class MistralAIService extends BaseService {
static MODULES = {
'@mistralai/mistralai': require('@mistralai/mistralai'),
}
/**
* Initializes the service's cost structure for different Mistral AI models.
* Sets up pricing information for various models including token costs for input/output.
* Each model entry specifies currency (usd-cents) and costs per million tokens.
* @private
*/
_construct () {
this.costs_ = {
'mistral-large-latest': {
@ -80,6 +95,12 @@ class MistralAIService extends BaseService {
},
};
}
/**
* Initializes the service's cost structure for different Mistral AI models.
* Sets up pricing information for various models including token costs for input/output.
* Each model entry specifies currency (USD cents) and costs per million tokens.
* @private
*/
async _init () {
const require = this.require;
const { Mistral } = require('@mistralai/mistralai');
@ -97,6 +118,13 @@ class MistralAIService extends BaseService {
// TODO: make this event-driven so it doesn't hold up boot
await this.populate_models_();
}
/**
* Populates the internal models array with available Mistral AI models and their configurations.
* Makes an API call to fetch model data, then processes and filters models based on cost information.
* Each model entry includes id, name, aliases, context window size, capabilities, and pricing.
* @private
* @returns {Promise<void>}
*/
async populate_models_ () {
const resp = await axios({
method: 'get',
@ -131,17 +159,41 @@ class MistralAIService extends BaseService {
}
// return resp.data;
}
/**
* Populates the internal models array with available Mistral AI models and their metadata
* Fetches model data from the API, filters based on cost configuration, and stores
* model objects containing ID, name, aliases, context length, capabilities, and pricing
* @private
* @async
* @returns {void}
*/
get_default_model () {
return 'mistral-large-latest';
}
static IMPLEMENTS = {
'puter-chat-completion': {
/**
* Implements the puter-chat-completion interface for MistralAI service
* Provides methods for listing models and generating chat completions
* @interface
* @property {Function} models - Returns array of available model details
* @property {Function} list - Returns array of model IDs
* @property {Function} complete - Generates chat completion with optional streaming
*/
async models () {
return this.models_array_;
},
/**
* Returns an array of available AI models with their details
* @returns {Promise<Array>} Array of model objects containing id, name, aliases, context window size, capabilities, and cost information
*/
async list () {
return this.models_array_.map(m => m.id);
},
/**
* Returns an array of model IDs supported by the MistralAI service
* @returns {Promise<string[]>} Array of model identifier strings
*/
async complete ({ messages, stream, model }) {
for ( let i = 0; i < messages.length; i++ ) {

84
src/backend/src/modules/puterai/OpenAICompletionService.js
View File

@ -1,17 +1,33 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PassThrough } = require('stream');
const APIError = require('../../api/APIError');
const BaseService = require('../../services/BaseService');
const { TypedValue } = require('../../services/drivers/meta/Runtime');
const { Context } = require('../../util/context');
const SmolUtil = require('../../util/smolutil');
const smol = require('@heyputer/putility').libs.smol;
const { nou } = require('../../util/langutil');
const { TeePromise } = require('../../util/promise');
const { TeePromise } = require('@heyputer/putility').libs.promise;
/**
* OpenAICompletionService class provides an interface to OpenAI's chat completion API.
* Extends BaseService to handle chat completions, message moderation, token counting,
* and streaming responses. Implements the puter-chat-completion interface and manages
* OpenAI API interactions with support for multiple models including GPT-4 variants.
* Handles usage tracking, spending records, and content moderation.
*/
class OpenAICompletionService extends BaseService {
static MODULES = {
openai: require('openai'),
tiktoken: require('tiktoken'),
}
/**
* Initializes the OpenAI service by setting up the API client with credentials
* and registering this service as a chat provider.
*
* @returns {Promise<void>} Resolves when initialization is complete
* @private
*/
async _init () {
const sk_key =
this.config?.openai?.secret_key ??
@ -28,10 +44,21 @@ class OpenAICompletionService extends BaseService {
});
}
/**
* Gets the default model identifier for OpenAI completions
* @returns {string} The default model ID 'gpt-4o-mini'
*/
get_default_model () {
return 'gpt-4o-mini';
}
/**
* Returns an array of available AI models with their pricing information.
* Each model object includes an ID and cost details (currency, tokens, input/output rates).
* @returns {Promise<Array<{id: string, cost: {currency: string, tokens: number, input: number, output: number}}>}
*/
async models_ () {
return [
{
@ -75,9 +102,25 @@ class OpenAICompletionService extends BaseService {
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implements the puter-chat-completion interface methods for model listing and chat completion
* @property {Object} models - Returns available AI models and their pricing
* @property {Function} list - Returns list of available model names/aliases
* @property {Function} complete - Handles chat completion requests with optional streaming
* @param {Object} params - Parameters for completion
* @param {Array} params.messages - Array of chat messages
* @param {boolean} params.test_mode - Whether to use test mode
* @param {boolean} params.stream - Whether to stream responses
* @param {string} params.model - Model ID to use
*/
async models () {
return await this.models_();
},
/**
* Retrieves a list of available AI models with their cost information
* @returns {Promise<Array>} Array of model objects containing id and cost details
* @private
*/
async list () {
const models = await this.models_();
const model_names = [];
@ -89,6 +132,10 @@ class OpenAICompletionService extends BaseService {
}
return model_names;
},
/**
* Lists all available model names including aliases
* @returns {Promise<string[]>} Array of model IDs and their aliases
*/
async complete ({ messages, test_mode, stream, model }) {
// for now this code (also in AIChatService.js) needs to be
@ -139,6 +186,14 @@ class OpenAICompletionService extends BaseService {
}
};
/**
* Checks text content against OpenAI's moderation API for inappropriate content
* @param {string} text - The text content to check for moderation
* @returns {Promise<Object>} Object containing flagged status and detailed results
* @property {boolean} flagged - Whether the content was flagged as inappropriate
* @property {Object} results - Raw moderation results from OpenAI API
*/
async check_moderation (text) {
// create moderation
const results = await this.openai.moderations.create({
@ -160,6 +215,17 @@ class OpenAICompletionService extends BaseService {
};
}
/**
* Completes a chat conversation using OpenAI's API
* @param {Array} messages - Array of message objects or strings representing the conversation
* @param {Object} options - Configuration options
* @param {boolean} options.stream - Whether to stream the response
* @param {boolean} options.moderation - Whether to perform content moderation
* @param {string} options.model - The model to use for completion
* @returns {Promise<Object>} The completion response containing message and usage info
* @throws {Error} If messages are invalid or content is flagged by moderation
*/
async complete (messages, { stream, moderation, model }) {
// Validate messages
if ( ! Array.isArray(messages) ) {
@ -234,7 +300,7 @@ class OpenAICompletionService extends BaseService {
if ( ! msg.content ) continue;
if ( typeof msg.content !== 'object' ) continue;
const content = SmolUtil.ensure_array(msg.content);
const content = smol.ensure_array(msg.content);
for ( const o of content ) {
if ( ! o.hasOwnProperty('image_url') ) continue;
@ -260,7 +326,7 @@ class OpenAICompletionService extends BaseService {
if ( ! msg.content ) continue;
if ( typeof msg.content !== 'object' ) continue;
const content = SmolUtil.ensure_array(msg.content);
const content = smol.ensure_array(msg.content);
for ( const o of content ) {
// console.log('part of content', o);
@ -275,8 +341,9 @@ class OpenAICompletionService extends BaseService {
const max_tokens = 4096 - token_count;
console.log('MAX TOKENS ???', max_tokens);
const svc_apiErrpr = this.services.get('api-error');
if ( max_tokens <= 8 ) {
throw APIError.create('max_tokens_exceeded', null, {
throw svc_apiErrpr.create('max_tokens_exceeded', {
input_tokens: token_count,
max_tokens: 4096 - 8,
});
@ -338,6 +405,13 @@ class OpenAICompletionService extends BaseService {
const spending_meta = {};
spending_meta.timestamp = Date.now();
spending_meta.count_tokens_input = token_count;
/**
* Records spending metadata for the chat completion request and performs token counting.
* Initializes metadata object with timestamp and token counts for both input and output.
* Uses tiktoken to count output tokens from the completion response.
* Records spending data via spending service and increments usage counters.
* @private
*/
spending_meta.count_tokens_output = (() => {
// count output tokens (overestimate)
const enc = this.modules.tiktoken.encoding_for_model(model);

24
src/backend/src/modules/puterai/OpenAIImageGenerationService.js
View File

@ -1,11 +1,26 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { Context } = require("../../util/context");
/**
* Service class for generating images using OpenAI's DALL-E API.
* Extends BaseService to provide image generation capabilities through
* the puter-image-generation interface. Supports different aspect ratios
* (square, portrait, landscape) and handles API authentication, request
* validation, and spending tracking.
*/
class OpenAIImageGenerationService extends BaseService {
static MODULES = {
openai: require('openai'),
}
/**
* Initializes the OpenAI client with API credentials from config
* @private
* @async
* @returns {Promise<void>}
*/
async _init () {
const sk_key =
this.config?.openai?.secret_key ??
@ -24,6 +39,15 @@ class OpenAIImageGenerationService extends BaseService {
}
},
['puter-image-generation']: {
/**
* Generates an image using OpenAI's DALL-E API
* @param {string} prompt - The text description of the image to generate
* @param {Object} options - Generation options
* @param {Object} options.ratio - Image dimensions ratio object with w/h properties
* @param {string} [options.model='dall-e-3'] - The model to use for generation
* @returns {Promise<string>} URL of the generated image
* @throws {Error} If prompt is not a string or ratio is invalid
*/
async generate ({ prompt, test_mode }) {
if ( test_mode ) {
return new TypedValue({

16
src/backend/src/modules/puterai/PuterAIModule.js
View File

@ -1,7 +1,23 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { AdvancedBase } = require("@heyputer/putility");
const config = require("../../config");
/**
* PuterAIModule class extends AdvancedBase to manage and register various AI services.
* This module handles the initialization and registration of multiple AI-related services
* including text processing, speech synthesis, chat completion, and image generation.
* Services are conditionally registered based on configuration settings, allowing for
* flexible deployment with different AI providers like AWS, OpenAI, Claude, Together AI,
* Mistral, Groq, and XAI.
* @extends AdvancedBase
*/
class PuterAIModule extends AdvancedBase {
/**
* Module for managing AI-related services in the Puter platform
* Extends AdvancedBase to provide core functionality
* Handles registration and configuration of various AI services like OpenAI, Claude, AWS services etc.
*/
async install (context) {
const services = context.get('services');

333
src/backend/src/modules/puterai/README.md Normal file
View File

@ -0,0 +1,333 @@
# PuterAIModule
PuterAIModule class extends AdvancedBase to manage and register various AI services.
This module handles the initialization and registration of multiple AI-related services
including text processing, speech synthesis, chat completion, and image generation.
Services are conditionally registered based on configuration settings, allowing for
flexible deployment with different AI providers like AWS, OpenAI, Claude, Together AI,
Mistral, Groq, and XAI.
## Services
### AIChatService
AIChatService class extends BaseService to provide AI chat completion functionality.
Manages multiple AI providers, models, and fallback mechanisms for chat interactions.
Handles model registration, usage tracking, cost calculation, content moderation,
and implements the puter-chat-completion driver interface. Supports streaming responses
and maintains detailed model information including pricing and capabilities.
#### Listeners
##### `boot.consolidation`
Handles consolidation during service boot by registering service aliases
and populating model lists/maps from providers.
Registers each provider as an 'ai-chat' service alias and fetches their
available models and pricing information. Populates:
- simple_model_list: Basic list of supported models
- detail_model_list: Detailed model info including costs
- detail_model_map: Maps model IDs/aliases to their details
#### Methods
##### `register_provider`
##### `moderate`
Moderates chat messages for inappropriate content using OpenAI's moderation service
###### Parameters
- **params:** The parameters object
- **params.messages:** Array of chat messages to moderate
##### `get_delegate`
Gets the appropriate delegate service for handling chat completion requests.
If the intended service is this service (ai-chat), returns undefined.
Otherwise returns the intended service wrapped as a puter-chat-completion interface.
##### `get_fallback_model`
Find an appropriate fallback model by sorting the list of models
by the euclidean distance of the input/output prices and selecting
the first one that is not in the tried list.
###### Parameters
- **param0:** null
##### `get_model_from_request`
### AIInterfaceService
Service class that manages AI interface registrations and configurations.
Handles registration of various AI services including OCR, chat completion,
image generation, and text-to-speech interfaces. Each interface defines
its available methods, parameters, and expected results.
#### Listeners
##### `driver.register.interfaces`
Service class for managing AI interface registrations and configurations.
Extends the base service to provide AI-related interface management.
Handles registration of OCR, chat completion, image generation, and TTS interfaces.
### AITestModeService
Service class that handles AI test mode functionality.
Extends BaseService to register test services for AI chat completions.
Used for testing and development of AI-related features by providing
a mock implementation of the chat completion service.
### AWSPollyService
AWSPollyService class provides text-to-speech functionality using Amazon Polly.
Extends BaseService to integrate with AWS Polly for voice synthesis operations.
Implements voice listing, speech synthesis, and voice selection based on language.
Includes caching for voice descriptions and supports both text and SSML inputs.
#### Methods
##### `describe_voices`
Describes available AWS Polly voices and caches the results
##### `synthesize_speech`
Synthesizes speech from text using AWS Polly
###### Parameters
- **text:** The text to synthesize
- **options:** Synthesis options
- **options.format:** Output audio format (e.g. 'mp3')
### AWSTextractService
AWSTextractService class - Provides OCR (Optical Character Recognition) functionality using AWS Textract
Extends BaseService to integrate with AWS Textract for document analysis and text extraction.
Implements driver capabilities and puter-ocr interface for document recognition.
Handles both S3-stored and buffer-based document processing with automatic region management.
#### Methods
##### `analyze_document`
Analyzes a document using AWS Textract to extract text and layout information
###### Parameters
- **file_facade:** Interface to access the document file
### ClaudeEnoughService
ClaudeEnoughService - A service class that implements a Claude-like AI interface
Extends XAIService to provide Claude-compatible responses while using alternative AI models.
Includes custom system prompts and model adaptation to simulate Claude's behavior
in the Puter platform's chat completion interface.
#### Methods
##### `get_system_prompt`
Service that emulates Claude's behavior using alternative AI models
##### `adapt_model`
### ClaudeService
ClaudeService class extends BaseService to provide integration with Anthropic's Claude AI models.
Implements the puter-chat-completion interface for handling AI chat interactions.
Manages message streaming, token limits, model selection, and API communication with Claude.
Supports system prompts, message adaptation, and usage tracking.
#### Methods
##### `get_default_model`
Returns the default model identifier for Claude API interactions
### FakeChatService
FakeChatService - A mock implementation of a chat service that extends BaseService.
Provides fake chat completion responses using Lorem Ipsum text generation.
Used for testing and development purposes when a real chat service is not needed.
Implements the 'puter-chat-completion' interface with list() and complete() methods.
### GroqAIService
Service class for integrating with Groq AI's language models.
Extends BaseService to provide chat completion capabilities through the Groq API.
Implements the puter-chat-completion interface for model management and text generation.
Supports both streaming and non-streaming responses, handles multiple models including
various versions of Llama, Mixtral, and Gemma, and manages usage tracking.
#### Methods
##### `get_default_model`
Returns the default model ID for the Groq AI service
### MistralAIService
MistralAIService class extends BaseService to provide integration with the Mistral AI API.
Implements chat completion functionality with support for various Mistral models including
mistral-large, pixtral, codestral, and ministral variants. Handles both streaming and
non-streaming responses, token usage tracking, and model management. Provides cost information
for different models and implements the puter-chat-completion interface.
#### Methods
##### `get_default_model`
Populates the internal models array with available Mistral AI models and their metadata
Fetches model data from the API, filters based on cost configuration, and stores
model objects containing ID, name, aliases, context length, capabilities, and pricing
### OpenAICompletionService
OpenAICompletionService class provides an interface to OpenAI's chat completion API.
Extends BaseService to handle chat completions, message moderation, token counting,
and streaming responses. Implements the puter-chat-completion interface and manages
OpenAI API interactions with support for multiple models including GPT-4 variants.
Handles usage tracking, spending records, and content moderation.
#### Methods
##### `get_default_model`
Gets the default model identifier for OpenAI completions
##### `check_moderation`
Checks text content against OpenAI's moderation API for inappropriate content
###### Parameters
- **text:** The text content to check for moderation
##### `complete`
Completes a chat conversation using OpenAI's API
###### Parameters
- **messages:** Array of message objects or strings representing the conversation
- **options:** Configuration options
- **options.stream:** Whether to stream the response
- **options.moderation:** Whether to perform content moderation
- **options.model:** The model to use for completion
### OpenAIImageGenerationService
Service class for generating images using OpenAI's DALL-E API.
Extends BaseService to provide image generation capabilities through
the puter-image-generation interface. Supports different aspect ratios
(square, portrait, landscape) and handles API authentication, request
validation, and spending tracking.
#### Methods
##### `generate`
### TogetherAIService
TogetherAIService class provides integration with Together AI's language models.
Extends BaseService to implement chat completion functionality through the
puter-chat-completion interface. Manages model listings, chat completions,
and streaming responses while handling usage tracking and model fallback testing.
#### Methods
##### `get_default_model`
Returns the default model ID for the Together AI service
### XAIService
XAIService class - Provides integration with X.AI's API for chat completions
Extends BaseService to implement the puter-chat-completion interface.
Handles model management, message adaptation, streaming responses,
and usage tracking for X.AI's language models like Grok.
#### Methods
##### `get_system_prompt`
Gets the system prompt used for AI interactions
##### `adapt_model`
##### `get_default_model`
Returns the default model identifier for the XAI service
## Notes
### Outside Imports
This module has external relative imports. When these are
removed it may become possible to move this module to an
extension.
**Imports:**
- `../../api/APIError`
- `../../services/auth/PermissionService`
- `../../services/BaseService` (use.BaseService)
- `../../services/database/consts`
- `../../services/drivers/meta/Construct`
- `../../services/drivers/meta/Runtime`
- `../../util/context`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../services/BaseService` (use.BaseService)
- `../../api/APIError`
- `../../services/BaseService` (use.BaseService)
- `../../util/langutil`
- `../../services/drivers/meta/Runtime`
- `../../api/APIError`
- `../../util/promise`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/langutil`
- `../../util/promise`
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/langutil`
- `../../util/promise`
- `../../api/APIError`
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/context`
- `../../util/smolutil`
- `../../util/langutil`
- `../../util/promise`
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/context`
- `../../config`
- `../../services/BaseService` (use.BaseService)
- `../../services/drivers/meta/Runtime`
- `../../util/langutil`
- `../../util/promise`
- `../../services/BaseService` (use.BaseService)
- `../../util/langutil`
- `../../services/drivers/meta/Runtime`
- `../../util/promise`

47
src/backend/src/modules/puterai/TogetherAIService.js
View File

@ -1,9 +1,18 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { PassThrough } = require("stream");
const BaseService = require("../../services/BaseService");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { nou } = require("../../util/langutil");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
/**
* TogetherAIService class provides integration with Together AI's language models.
* Extends BaseService to implement chat completion functionality through the
* puter-chat-completion interface. Manages model listings, chat completions,
* and streaming responses while handling usage tracking and model fallback testing.
* @extends BaseService
*/
class TogetherAIService extends BaseService {
static MODULES = {
['together-ai']: require('together-ai'),
@ -11,6 +20,13 @@ class TogetherAIService extends BaseService {
uuidv4: require('uuid').v4,
}
/**
* Initializes the TogetherAI service by setting up the API client and registering as a chat provider
* @async
* @returns {Promise<void>}
* @private
*/
async _init () {
const require = this.require;
const Together = require('together-ai');
@ -27,20 +43,41 @@ class TogetherAIService extends BaseService {
});
}
/**
* Returns the default model ID for the Together AI service
* @returns {string} The ID of the default model (meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo)
*/
get_default_model () {
return 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo';
}
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implements the puter-chat-completion interface for TogetherAI service
* Contains methods for listing models and generating chat completions
* @property {Object} models - Method to get available models
* @property {Object} list - Method to get list of model IDs
* @property {Object} complete - Method to generate chat completions
*/
async models () {
return await this.models_();
},
/**
* Retrieves available AI models from the Together API
* @returns {Promise<Array>} Array of model objects with their properties
* @implements {puter-chat-completion.models}
*/
async list () {
let models = this.modules.kv.get(`${this.kvkey}:models`);
if ( ! models ) models = await this.models_();
return models.map(model => model.id);
},
/**
* Lists available AI model IDs from the cache or fetches them if not cached
* @returns {Promise<string[]>} Array of model ID strings
*/
async complete ({ messages, stream, model }) {
if ( model === 'model-fallback-test-1' ) {
throw new Error('Model Fallback Test 1');
@ -103,6 +140,14 @@ class TogetherAIService extends BaseService {
}
}
/**
* Fetches and caches available AI models from Together API
* @private
* @returns {Promise<Array>} Array of model objects containing id, name, context length,
* description and pricing information
* @remarks Models are cached for 5 minutes in KV store
*/
async models_ () {
let models = this.modules.kv.get(`${this.kvkey}:models`);
if ( models ) return models;

51
src/backend/src/modules/puterai/XAIService.js
View File

@ -1,9 +1,10 @@
// METADATA // {"ai-commented":{"service":"claude"}}
const { default: Anthropic } = require("@anthropic-ai/sdk");
const BaseService = require("../../services/BaseService");
const { whatis, nou } = require("../../util/langutil");
const { PassThrough } = require("stream");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { TeePromise } = require("../../util/promise");
const { TeePromise } = require('@heyputer/putility').libs.promise;
const PUTER_PROMPT = `
You are running on an open-source platform called Puter,
@ -14,11 +15,24 @@ const PUTER_PROMPT = `
user of the driver interface (typically an app on Puter):
`.replace('\n', ' ').trim();
/**
* XAIService class - Provides integration with X.AI's API for chat completions
* Extends BaseService to implement the puter-chat-completion interface.
* Handles model management, message adaptation, streaming responses,
* and usage tracking for X.AI's language models like Grok.
* @extends BaseService
*/
class XAIService extends BaseService {
static MODULES = {
openai: require('openai'),
}
/**
* Gets the system prompt used for AI interactions
* @returns {string} The base system prompt that identifies the AI as running on Puter
*/
get_system_prompt () {
return PUTER_PROMPT;
}
@ -27,6 +41,12 @@ class XAIService extends BaseService {
return model;
}
/**
* Initializes the XAI service by setting up the OpenAI client and registering with the AI chat provider
* @private
* @returns {Promise<void>} Resolves when initialization is complete
*/
async _init () {
this.openai = new this.modules.openai.OpenAI({
apiKey: this.global_config.services.xai.apiKey,
@ -40,15 +60,30 @@ class XAIService extends BaseService {
});
}
/**
* Returns the default model identifier for the XAI service
* @returns {string} The default model ID 'grok-beta'
*/
get_default_model () {
return 'grok-beta';
}
static IMPLEMENTS = {
['puter-chat-completion']: {
/**
* Implements the interface for the puter-chat-completion driver
* Contains methods for listing models, getting model details,
* and handling chat completions with streaming support
* @type {Object}
*/
async models () {
return await this.models_();
},
/**
* Returns a list of available AI models with their capabilities and pricing details
* @returns {Promise<Array>} Array of model objects containing id, name, context window size, and cost information
*/
async list () {
const models = await this.models_();
const model_names = [];
@ -60,6 +95,10 @@ class XAIService extends BaseService {
}
return model_names;
},
/**
* Returns a list of all available model names including their aliases
* @returns {Promise<string[]>} Array of model names and their aliases
*/
async complete ({ messages, stream, model }) {
model = this.adapt_model(model);
const adapted_messages = [];
@ -162,6 +201,16 @@ class XAIService extends BaseService {
}
}
/**
* Retrieves available AI models and their specifications
* @returns {Promise<Array>} Array of model objects containing:
* - id: Model identifier string
* - name: Human readable model name
* - context: Maximum context window size
* - cost: Pricing information object with currency and rates
* @private
*/
async models_ () {
return [
{

77
src/backend/src/modules/puterai/lib/AsModeration.js Normal file
View File

@ -0,0 +1,77 @@
const { default: dedent } = require("dedent");
class AsModeration {
constructor ({ chat, model }) {
this.chat = chat;
this.model = model;
}
async moderate (text) {
// copied from Anthropic documentation
// https://docs.anthropic.com/en/docs/about-claude/use-case-guides/content-moderation
const unsafe_categories = [
'Child Exploitation',
'Conspiracy Theories',
'Hate',
'Indiscriminate Weapons',
'Intellectual Property',
'Non-Violent Crimes',
'Privacy',
'Self-Harm',
'Sex Crimes',
'Sexual Content',
'Specialized Advice',
'Violent Crimes'
];
const unsafe_category_str = unsafe_categories.join('\n');
const assessment_prompt = dedent(`
Determine whether the following message warrants moderation,
based on the unsafe categories outlined below.
Message:
<message>
${text
.replace('<', '&lt;')
.replace('>', '&gt;')
}
</message>
Unsafe Categories:
<categories>
${unsafe_category_str}
</categories>
Respond with ONLY a JSON object, using the format below:
{{
"violation": <Boolean field denoting whether the message should be moderated>,
"categories": [Comma-separated list of violated categories],
"explanation": [Optional. Only include if there is a violation.]
}}
`);
const result = await this.chat.complete({
messages: [
{
role: 'user',
content: assessment_prompt,
}
]
});
console.log('result???', require('util').inspect(result, { depth: null }));
const str = result.message?.content?.[0]?.text ??
result.messages?.[0]?.content?.[0]?.text ??
'{ "violation": true }';
const parsed = JSON.parse(str);
console.log('parsed?', parsed);
return ! parsed.violation;
}
}
module.exports = {
AsModeration,
};

17
src/backend/src/filesystem/storage/DatabaseFSEntryFetcher.js → src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js
View File

@ -17,16 +17,11 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const { DB_READ } = require("../../services/database/consts");
const { abtest } = require("../../util/otelutil");
const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeChildSelector, RootNodeSelector } = require("../node/selectors");
module.exports = class DatabaseFSEntryFetcher {
constructor ({ services }) {
this.services = services;
this.log = services.get('log-service').create('DatabaseFSEntryFetcher');
this.db = services.get('database').get(DB_READ, 'filesystem');
const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeChildSelector, RootNodeSelector } = require("../../filesystem/node/selectors");
const BaseService = require("../../services/BaseService");
module.exports = class DatabaseFSEntryFetcher extends BaseService {
_construct () {
this.defaultProperties = [
'id',
'associated_app_id',
@ -57,6 +52,10 @@ module.exports = class DatabaseFSEntryFetcher {
]
}
_init () {
this.db = this.services.get('database').get(DB_READ, 'filesystem');
}
async find (selector, fetch_entry_options) {
if ( selector instanceof RootNodeSelector ) {
return selector.entry;

Some files were not shown because too many files have changed in this diff Show More