mirror of
https://github.com/cocktailpeanut/dalai
synced 2024-11-20 23:07:32 +01:00
426 lines
14 KiB
JavaScript
426 lines
14 KiB
JavaScript
const os = require('os');
|
|
const pty = require('node-pty');
|
|
const git = require('isomorphic-git');
|
|
const http = require('isomorphic-git/http/node');
|
|
const Http = require("http")
|
|
const path = require('path');
|
|
const fs = require("fs");
|
|
const tar = require('tar');
|
|
const { createServer } = require("http");
|
|
const { Server } = require("socket.io");
|
|
const { io } = require("socket.io-client");
|
|
const term = require( 'terminal-kit' ).terminal;
|
|
const Downloader = require("nodejs-file-downloader");
|
|
const semver = require('semver');
|
|
const _7z = require('7zip-min');
|
|
const axios = require('axios')
|
|
const platform = os.platform()
|
|
const shell = platform === 'win32' ? 'powershell.exe' : 'bash';
|
|
const L = require("./llama")
|
|
const A = require("./alpaca")
|
|
class Dalai {
|
|
constructor(home) {
|
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
//
|
|
// 1. manually set llama.cpp home
|
|
// 2. otherwise store llama.cpp at ~/llama.cpp
|
|
//
|
|
// # NOTE
|
|
// Could have used process.cwd() (The current execution directory) to download llama.cpp
|
|
// but this makes it cumbersome as you try to build multiple apps, because by default Dalai client will
|
|
// look for the current execution directory for llama.cpp.
|
|
// It's simpler to set the ~/llama.cpp as the default directory and use that path as the single source
|
|
// of truth and let multiple apps all connect to that path
|
|
// Otherwise if you want to customize the path you can just pass in the "home" attribute to manually set it.
|
|
//
|
|
////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
this.home = home ? path.resolve(home) : path.resolve(os.homedir(), "dalai")
|
|
try {
|
|
fs.mkdirSync(this.home, { recursive: true })
|
|
} catch (e) { }
|
|
this.config = {
|
|
name: 'xterm-color',
|
|
cols: 200,
|
|
rows: 30,
|
|
}
|
|
this.cores = {
|
|
llama: new L(this),
|
|
alpaca: new A(this),
|
|
}
|
|
}
|
|
down(url, dest, headers) {
|
|
return new Promise((resolve, reject) => {
|
|
const task = path.basename(dest)
|
|
this.startProgress(task)
|
|
axios({
|
|
url,
|
|
method: 'GET',
|
|
responseType: 'stream',
|
|
maxContentLength: Infinity,
|
|
headers,
|
|
onDownloadProgress: progressEvent => {
|
|
const progress = (progressEvent.loaded / progressEvent.total) * 100;
|
|
this.progress(task, progress)
|
|
}
|
|
|
|
}).then(response => {
|
|
const writer = fs.createWriteStream(dest);
|
|
response.data.pipe(writer);
|
|
writer.on('finish', () => {
|
|
this.progressBar.update(1);
|
|
term("\n")
|
|
resolve()
|
|
});
|
|
}).catch(error => {
|
|
reject(error)
|
|
});
|
|
})
|
|
}
|
|
async python () {
|
|
// install self-contained python => only for windows for now
|
|
// 1. download
|
|
// 2. unzip
|
|
|
|
const filename = "cpython-3.10.9+20230116-x86_64-pc-windows-msvc-shared-install_only.tar.gz"
|
|
const task = "downloading self contained python"
|
|
const downloader = new Downloader({
|
|
url: `https://github.com/indygreg/python-build-standalone/releases/download/20230116/${filename}`,
|
|
directory: this.home,
|
|
onProgress: (percentage, chunk, remainingSize) => {
|
|
this.progress(task, percentage)
|
|
},
|
|
});
|
|
try {
|
|
await this.startProgress(task)
|
|
await downloader.download();
|
|
} catch (error) {
|
|
console.log(error);
|
|
}
|
|
this.progressBar.update(1);
|
|
console.log("extracting python")
|
|
await tar.x({
|
|
file: path.resolve(this.home, filename),
|
|
C: this.home,
|
|
strict: true
|
|
})
|
|
console.log("cleaning up temp files")
|
|
await fs.promises.rm(path.resolve(this.home, filename))
|
|
}
|
|
async mingw() {
|
|
const mingw = "https://github.com/niXman/mingw-builds-binaries/releases/download/12.2.0-rt_v10-rev2/x86_64-12.2.0-release-win32-seh-msvcrt-rt_v10-rev2.7z"
|
|
const downloader = new Downloader({
|
|
url: mingw,
|
|
directory: this.home,
|
|
onProgress: (percentage, chunk, remainingSize) => {
|
|
this.progress("download mingw", percentage)
|
|
},
|
|
});
|
|
try {
|
|
await this.startProgress("download mingw")
|
|
await downloader.download();
|
|
} catch (error) {
|
|
console.log(error);
|
|
}
|
|
this.progressBar.update(1);
|
|
await new Promise((resolve, reject) => {
|
|
_7z.unpack(path.resolve(this.home, "x86_64-12.2.0-release-win32-seh-msvcrt-rt_v10-rev2.7z"), this.home, (err) => {
|
|
if (err) {
|
|
reject(err)
|
|
} else {
|
|
resolve()
|
|
}
|
|
})
|
|
})
|
|
console.log("cleaning up temp files")
|
|
await fs.promises.rm(path.resolve(this.home, "x86_64-12.2.0-release-win32-seh-msvcrt-rt_v10-rev2.7z"))
|
|
}
|
|
async query(req, cb) {
|
|
|
|
console.log(`> query:`, req)
|
|
if (req.method === "installed") {
|
|
let models = await this.installed()
|
|
for(let model of models) {
|
|
cb(model)
|
|
}
|
|
cb('\n\n<end>')
|
|
return
|
|
}
|
|
|
|
|
|
const [Core, Model] = req.model.split(".")
|
|
|
|
console.log( { Core, Model } )
|
|
|
|
let o = {
|
|
seed: req.seed || -1,
|
|
threads: req.threads || 8,
|
|
n_predict: req.n_predict || 128,
|
|
model: `models/${Model || "7B"}/ggml-model-q4_0.bin`,
|
|
}
|
|
|
|
if (!fs.existsSync(path.resolve(this.home, Core, "models", Model))) {
|
|
cb(`File does not exist: ${Model}. Try "dalai ${Core} get ${Model}" first.`)
|
|
return
|
|
}
|
|
|
|
if (req.top_k) o.top_k = req.top_k
|
|
if (req.top_p) o.top_p = req.top_p
|
|
if (req.temp) o.temp = req.temp
|
|
if (req.batch_size) o.batch_size = req.batch_size
|
|
if (req.repeat_last_n) o.repeat_last_n = req.repeat_last_n
|
|
if (req.repeat_penalty) o.repeat_penalty = req.repeat_penalty
|
|
if (typeof req.interactive !== "undefined") o.interactive = req.interactive
|
|
|
|
let chunks = []
|
|
for(let key in o) {
|
|
chunks.push(`--${key} ${o[key]}`)
|
|
}
|
|
chunks.push(`-p "${req.prompt}"`)
|
|
|
|
const main_bin_path = platform === "win32" ? path.resolve(this.home, Core, "build", "Release", "llama") : path.resolve(this.home, Core, "main")
|
|
if (req.full) {
|
|
await this.exec(`${main_bin_path} ${chunks.join(" ")}`, this.cores[Core].home, cb)
|
|
} else {
|
|
const startpattern = /.*sampling parameters:.*/g
|
|
const endpattern = /.*mem per token.*/g
|
|
let started = req.debug
|
|
let ended = false
|
|
let writeEnd = !req.skip_end
|
|
await this.exec(`${main_bin_path} ${chunks.join(" ")}`, this.cores[Core].home, (msg) => {
|
|
if (endpattern.test(msg)) ended = true
|
|
if (started && !ended) {
|
|
cb(msg)
|
|
} else if (ended && writeEnd) {
|
|
cb('\n\n<end>')
|
|
writeEnd = false
|
|
}
|
|
if (startpattern.test(msg)) started = true
|
|
})
|
|
}
|
|
}
|
|
async get(core, ...models) {
|
|
let res = await this.cores[core].get(...models)
|
|
return res
|
|
}
|
|
async installed() {
|
|
// get cores
|
|
const modelNames = []
|
|
for(let core of ["alpaca", "llama"]) {
|
|
const modelsPath = path.resolve(this.home, core, "models")
|
|
console.log("modelsPath", modelsPath)
|
|
let modelFolders = []
|
|
try {
|
|
modelFolders = (await fs.promises.readdir(modelsPath, { withFileTypes: true }))
|
|
.filter(dirent => dirent.isDirectory())
|
|
.map(dirent => dirent.name)
|
|
} catch (e) {
|
|
}
|
|
|
|
console.log({ modelFolders })
|
|
for(let modelFolder of modelFolders) {
|
|
if (fs.existsSync(path.resolve(modelsPath, modelFolder, 'ggml-model-q4_0.bin'))) {
|
|
modelNames.push(`${core}.${modelFolder}`)
|
|
console.log("exists", modelFolder)
|
|
}
|
|
}
|
|
}
|
|
return modelNames
|
|
}
|
|
async install (core) {
|
|
/**************************************************************************************************************
|
|
*
|
|
* 2. Download Core
|
|
*
|
|
**************************************************************************************************************/
|
|
let engine = this.cores[core]
|
|
let exists = s => new Promise(r=>fs.access(s, fs.constants.F_OK, e => r(!e)))
|
|
let e = await exists(path.resolve(engine.home));
|
|
if (e) {
|
|
console.log("try fetching", engine.home, engine.url)
|
|
await git.fetch({ fs, http, dir: engine.home, url: engine.url })
|
|
} else {
|
|
console.log("try cloning", engine.home, engine.url)
|
|
await git.clone({ fs, http, dir: engine.home, url: engine.url })
|
|
}
|
|
console.log("next", core, engine.make);
|
|
/**************************************************************************************************************
|
|
*
|
|
* 4. Compile & Build
|
|
* - make: linux + mac
|
|
* - cmake: windows
|
|
*
|
|
**************************************************************************************************************/
|
|
await engine.make()
|
|
}
|
|
async setup() {
|
|
|
|
let success;
|
|
|
|
/**************************************************************************************************************
|
|
*
|
|
* 1. Validate
|
|
*
|
|
**************************************************************************************************************/
|
|
// Check if current version is greater than or equal to 18
|
|
const node_version = process.version;
|
|
if (!semver.gte(node_version, '18.0.0')) {
|
|
throw new Error("outdated Node version, please install Node 18 or newer")
|
|
}
|
|
|
|
/**************************************************************************************************************
|
|
*
|
|
* 3. Download Global Dependencies
|
|
* - Python (windows only)
|
|
* - build-essential (linux only)
|
|
* - virtualenv
|
|
* - torch, numpy, etc.
|
|
*
|
|
**************************************************************************************************************/
|
|
|
|
// 3.1. Python: Windows doesn't ship with python, so install a dedicated self-contained python
|
|
if (platform === "win32") {
|
|
await this.python()
|
|
}
|
|
const root_python_paths = (platform === "win32" ? [path.resolve(this.home, "python", "python.exe")] : ["python3", "python"])
|
|
const root_pip_paths = (platform === "win32" ? [path.resolve(this.home, "python", "python -m pip")] : ["pip3", "pip"])
|
|
|
|
// 3.2. Build tools
|
|
if (platform === "linux") {
|
|
// ubuntu debian
|
|
success = await this.exec("apt-get install build-essential python3-venv -y")
|
|
if (!success) {
|
|
// fefdora
|
|
await this.exec("dnf install make automake gcc gcc-c++ kernel-devel python3-virtualenv -y")
|
|
}
|
|
} else {
|
|
// for win32 / darwin
|
|
for(let root_pip_path of root_pip_paths) {
|
|
success = await this.exec(`${root_pip_path} install --user virtualenv`)
|
|
if (success) break;
|
|
}
|
|
if (!success) {
|
|
throw new Error("cannot install virtualenv")
|
|
}
|
|
|
|
}
|
|
|
|
// 3.3. virtualenv
|
|
const venv_path = path.join(this.home, "venv")
|
|
for(let root_python_path of root_python_paths) {
|
|
success = await this.exec(`${root_python_path} -m venv ${venv_path}`)
|
|
if (success) break;
|
|
}
|
|
if (!success) {
|
|
throw new Error("cannot execute python3 or python")
|
|
return
|
|
}
|
|
|
|
// 3.4. Python libraries
|
|
const pip_path = platform === "win32" ? path.join(venv_path, "Scripts", "pip.exe") : path.join(venv_path, "bin", "pip")
|
|
const python_path = platform == "win32" ? path.join(venv_path, "Scripts", "python.exe") : path.join(venv_path, 'bin', 'python')
|
|
// cmake (only on windows. the rest platforms use make)
|
|
if (platform === "win32") {
|
|
success = await this.exec(`${pip_path} install cmake`)
|
|
if (!success) {
|
|
throw new Error("cmake installation failed")
|
|
return
|
|
}
|
|
}
|
|
success = await this.exec(`${pip_path} install --upgrade pip setuptools wheel`)
|
|
if (!success) {
|
|
throw new Error("pip setuptools wheel upgrade failed")
|
|
return
|
|
}
|
|
success = await this.exec(`${pip_path} install torch torchvision torchaudio sentencepiece numpy`)
|
|
//success = await this.exec(`${pip_path} install torch torchvision torchaudio sentencepiece numpy wget`)
|
|
if (!success) {
|
|
throw new Error("dependency installation failed")
|
|
return
|
|
}
|
|
|
|
|
|
}
|
|
serve(port) {
|
|
const httpServer = createServer();
|
|
const io = new Server(httpServer)
|
|
io.on("connection", (socket) => {
|
|
socket.on('request', async (req) => {
|
|
await this.query(req, (str) => {
|
|
io.emit("result", { response: str, request: req })
|
|
})
|
|
});
|
|
});
|
|
httpServer.listen(port)
|
|
}
|
|
http(httpServer) {
|
|
const io = new Server(httpServer)
|
|
io.on("connection", (socket) => {
|
|
socket.on('request', async (req) => {
|
|
await this.query(req, (str) => {
|
|
io.emit("result", { response: str, request: req })
|
|
})
|
|
});
|
|
});
|
|
}
|
|
async request(req, cb) {
|
|
if (req.url) {
|
|
await this.connect(req, cb)
|
|
} else {
|
|
await this.query(req, cb)
|
|
}
|
|
}
|
|
connect(req, cb) {
|
|
const socket = io(req.url)
|
|
socket.emit('request', req)
|
|
socket.on('response', cb)
|
|
socket.on('error', function(e) {
|
|
throw e
|
|
});
|
|
}
|
|
exec(cmd, cwd, cb) {
|
|
return new Promise((resolve, reject) => {
|
|
const config = Object.assign({}, this.config)
|
|
if (cwd) {
|
|
config.cwd = path.resolve(cwd)
|
|
}
|
|
console.log(`exec: ${cmd} in ${config.cwd}`)
|
|
const ptyProcess = pty.spawn(shell, [], config)
|
|
ptyProcess.onData((data) => {
|
|
if (cb) {
|
|
cb(data)
|
|
} else {
|
|
process.stdout.write(data);
|
|
}
|
|
});
|
|
ptyProcess.onExit((res) => {
|
|
if (res.exitCode === 0) {
|
|
// successful
|
|
resolve(true)
|
|
} else {
|
|
// something went wrong
|
|
resolve(false)
|
|
}
|
|
});
|
|
ptyProcess.write(`${cmd}\r`)
|
|
ptyProcess.write("exit\r")
|
|
})
|
|
}
|
|
progress(task, percent) {
|
|
this.progressBar.update(percent/100);
|
|
//if (percent >= 100) {
|
|
// setTimeout(() => {
|
|
// term("\n")
|
|
// }, 200)
|
|
//}
|
|
}
|
|
startProgress(title) {
|
|
this.progressBar = term.progressBar({
|
|
width: 120,
|
|
title,
|
|
eta: true ,
|
|
percent: true
|
|
});
|
|
}
|
|
}
|
|
module.exports = Dalai
|