init

2025-03-06 18:53:01 +01:00 · 2023-03-12 16:06:51 -04:00 · 2023-03-12 16:06:51 -04:00 · 38a3eca656
commit 38a3eca656
20 changed files with 2129 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,3 @@
+.env
+.DS_Store
+node_modules
--- a/bin/cli.js
+++ b/bin/cli.js
@ -0,0 +1,44 @@
+#! /usr/bin/env node
+const Dalai = require("../index")
+const Web = require("./web/index")
+if (process.argv.length > 0) {
+  let [cmd, ...args] = process.argv.slice(2)
+  if (cmd === "serve") {
+    const port = (args.length > 0 ? parseInt(args[0]) : 3000)
+    Web(port)
+  } else if (cmd === "install") {
+    if (args.length === 0) args = ["7B"]
+    for(let arg of args) {
+      if (!["7B", "13B",  "30B", "65B"].includes(arg)) {
+        console.log(`##########################################################
+#
+#   ERROR
+#   The arguments must be one or more of the following:
+# 
+#   7B, 13B, 30B, 65B
+#
+##########################################################
+
+[Example]
+
+# install just 7B (default)
+npx dalai install   
+
+# install 7B manually
+npx dalai install 7B
+
+# install 7B and 13B
+npx dalai install 7B 13B
+`)
+        process.exit(1)
+        break;
+      }
+    }
+    new Dalai().install(...args).then(() => {
+      process.exit(1)
+    })
+  }
+} else {
+  console.log("ERROR: Please pass a command")
+  process.exit(1)
+}
--- a/bin/web/index.js
+++ b/bin/web/index.js
@ -0,0 +1,22 @@
+const express = require('express')
+const http = require('http')
+const path = require('path')
+const Dalai = require("../../index")
+const app = express()
+const httpServer = http.Server(app);
+const dalai = new Dalai()
+const start = (port) => {
+  dalai.http(httpServer)
+  app.use(express.static(path.resolve(__dirname, 'public')))
+  app.use(express.json());
+  app.use(express.urlencoded());
+  app.set('view engine', 'ejs');
+  app.set('views', path.resolve(__dirname, "views"))
+  app.get("/", (req, res) => {
+    res.render("index")
+  })
+  httpServer.listen(port, () => {
+    console.log("started server")
+  })
+}
+module.exports = start
--- a/bin/web/public/socket.io.min.js
+++ b/bin/web/public/socket.io.min.js
--- a/bin/web/views/index.ejs
+++ b/bin/web/views/index.ejs
@ -0,0 +1,107 @@
+<html>
+<head>
+<title>Dalai LLaMA</title>
+<style>
+  body { margin: 0; padding: 10px; color: rgba(0,0,0,0.8); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; }
+  form { border: 2px solid rgba(0,0,0,0.2); display: flex; padding: 5px; box-sizing: border-box; margin: 0; }
+  #input { white-space: pre-wrap; padding: 5px; outline: none; border: none; flex-grow: 1;  font-size: 14px; box-sizing: border-box; }
+  #input:focus { outline: none; }
+  #form > button { padding: 10px; background: gold; border: none; border-radius: 3px; outline: none; color: black; box-sizing: border-box; }
+
+  #messages { list-style-type: none; margin: 0; box-sizing: border-box; font-size: 14px; padding: 0; }
+  #messages > li { padding: 10px; font-size: 14px; box-sizing: border-box; }
+  #messages > li:nth-child(odd) { background: #efefef; }
+  li { white-space: pre-wrap; }
+  .loading {
+    padding: 10px;
+    box-sizing: border-box;
+    color: rgba(0,0,0,0.7);
+    font-size: 14px;
+    background: #efefef;
+  }
+  .hidden {
+    display: none !important;
+  }
+  .info {
+    font-size: 12px;
+    padding: 5px;
+  }
+</style>
+</head>
+<body>
+<form id="form" action="">
+  <div contenteditable id='input'></div>
+  <button>autocomplete</button>
+</form>
+<div class='info'>TIP: shift+enter for multiple lines</div>
+<div class='loading hidden'></div>
+<ul id="messages"></ul>
+<script src="/socket.io.min.js"></script>
+<script>
+const socket = io();
+const form = document.getElementById('form');
+const input = document.querySelector('#input');
+const loading = (on) => {
+  if (on) {
+    document.querySelector(".loading").textContent = on
+    document.querySelector(".loading").classList.remove("hidden")
+  } else {
+    document.querySelector(".loading").textContent = ""
+    document.querySelector(".loading").classList.add("hidden")
+  }
+}
+form.addEventListener('submit', (e) => {
+  e.preventDefault();
+  e.stopPropagation()
+  if (input.textContent) {
+    socket.emit('request', {
+      prompt: input.textContent,
+      n_predict: 256
+    })
+    loading(input.textContent)
+    input.textContent = "";
+  }
+});
+input.addEventListener("keydown", (e) => {
+  console.log("e", e)
+  if (e.keyCode === 13) {
+    e.preventDefault();
+    if (e.shiftKey) {
+      document.execCommand("insertLineBreak");
+    } else {
+      form.requestSubmit()
+    }
+  }
+})
+
+const sha256 = async (input) => {
+  const textAsBuffer = new TextEncoder().encode(input);
+  const hashBuffer = await window.crypto.subtle.digest("SHA-256", textAsBuffer);
+  const hashArray = Array.from(new Uint8Array(hashBuffer));
+  const hash = hashArray
+    .map((item) => item.toString(16).padStart(2, "0"))
+    .join("");
+  return hash;
+};
+const say = (msg, id) => {
+  let item = document.createElement('li');
+  if (id) item.setAttribute("data-id", id)
+  item.textContent = msg;
+  messages.prepend(item);
+}
+socket.on('result', async ({ request, response }) => {
+  loading(false)
+  const id = await sha256(request.prompt) 
+  console.log({ id, prompt: request.prompt })
+
+  let existing = document.querySelector(`[data-id='${id}']`)
+  if (existing) {
+    existing.textContent = existing.textContent + response
+  } else {
+    say(response, id)
+  }
+//  window.scrollTo(0, document.body.scrollHeight);
+});
+</script>
+</body>
+</html>
--- a/demo/client.js
+++ b/demo/client.js
@ -0,0 +1,9 @@
+const Dalai = require('../index')
+new Dalai("ws://localhost:3000").request({
+new Dalai().request({
+  model: "7B",
+  prompt: "If aliens were actually time travlers from the future,",
+  n_predict: 400
+}, (msg) => {
+  process.stdout.write(msg)
+})
--- a/demo/debug.js
+++ b/demo/debug.js
@ -0,0 +1,10 @@
+// Return the full response (not just the answer) => need to pass "full: true" in the request
+const Dalai = require('../index')
+new Dalai().request({
+  full: true,
+  model: "7B",
+  prompt: "If aliens were actually time travlers from the future,",
+  n_predict: 400
+}, (msg) => {
+  process.stdout.write(msg)
+})
--- a/demo/install.js
+++ b/demo/install.js
@ -0,0 +1,5 @@
+const L = require("../index")
+const l = new L();
+(async () => {
+  await l.install("7B")
+})();
--- a/demo/localrequest.js
+++ b/demo/localrequest.js
@ -0,0 +1,8 @@
+const Dalai = require('../index')
+new Dalai().request({
+  model: "7B",
+  prompt: "If aliens were actually time travlers from the future,",
+  n_predict: 400
+}, (msg) => {
+  process.stdout.write(msg)
+})
--- a/demo/query.js
+++ b/demo/query.js
@ -0,0 +1,21 @@
+const Dalai = require("../index")
+const dalai = new Dalai();
+const prompts = [
+  `Here's a sentence:
+
+I doesn't know how he is a president of the united states
+
+A more sophisticated and grammatically correct version of above sentence would be:`,
+  `here are some crazy ideas for an app that uses AI`,
+  `1, 2, 3, 5, 8, 13, 21,`,
+  `The following is a sequence of notes from a jazz improvisation:`
+];
+(async () => {
+  await dalai.request({
+    model: "7B",
+    prompt: prompts[3],
+    n_predict: 1000,
+  }, (str) => {
+    process.stdout.write(str)
+  })
+})();
--- a/demo/server.js
+++ b/demo/server.js
@ -0,0 +1,2 @@
+const Dalai = require("../index")
+new Dalai().serve(3000)
--- a/docs/README.md
+++ b/docs/README.md
@ -0,0 +1,212 @@
+# Dalai
+
+Dead simple way to run LLaMA on your computer.
+
+<a href="https://github.com/cocktailpeanut/dalai" class='inverse btn'><i class="fa-brands fa-github"></i> Github</a>
+<a href="https://twitter.com/cocktailpeanut" class='inverse btn'><i class="fa-brands fa-twitter"></i> Twitter</a>
+
+---
+
+1. Powered by [llama.cpp](https://github.com/ggerganov/llama.cpp) and [llama-dl CDN](https://github.com/shawwn/llama-dl)
+2. Web app included
+3. Super simple JavaScript API
+
+![dalai.gif](dalai.gif)
+
+---
+
+# Install
+
+Basic install (7B model only)
+
+```
+npx dalai install
+```
+
+Install all models
+
+```
+npx dalai install 7B 13B 30B 65B
+```
+
+The install command :
+
+1. Creates a folder named `dalai` under your home directory (`~`)
+2. Installs and builds the [llama.cpp](https://github.com/ggerganov/llama.cpp) project under `~/dalai`
+3. Downloads all the requested models from the [llama-dl CDN](https://github.com/shawwn/llama-dl) to `~/dalai/models`
+4. Runs some tasks to convert the LLaMA models so they can be used
+
+---
+
+# Quickstart
+
+Install the 7B model (default) and start a web UI:
+
+```
+npx dalai install
+npx dalai serve
+```
+
+Then go to http://localhost:3000
+
+Above two commands do the following:
+
+1. First installs the 7B module (default)
+2. Then starts a web/API server at port 3000
+
+---
+
+# API
+
+Dalai is also an NPM package:
+
+1. programmatically install
+2. locally make requests to the model
+3. run a dalai server (powered by socket.io)
+3. programmatically make requests to a remote dalai server (via socket.io)
+
+Dalai is an NPM package. You can install it using:
+
+```
+npm install dalai
+```
+
+---
+
+## 1. constructor()
+
+### Syntax
+
+```javascript
+const dalai = new Dalai(url)
+```
+
+- `url`: (optional)
+  - if unspecified, it uses the node.js API to directly run dalai
+  - if specified (for example `ws://localhost:3000`) it looks for a socket.io endpoint at the URL and connects to it.
+
+### Examples
+
+Initializing a client that connects to a local model (no network):
+
+```javascript
+const dalai = new Dalai()
+```
+
+Initializing a client that connects to a remote dalai server (a dalai server must be running at the URL):
+
+```javascript
+const dalai = new Dalai("ws://localhost:3000")
+```
+
+---
+
+## 2. request()
+
+### Syntax
+
+```javascript
+dalai.request(req, callback)
+```
+
+- `req`: a request object. made up of the following attributes:
+  - `prompt`: **(required)** The prompt string
+  - `model`: **(required)** The model name to query ("7B", "13B", etc.)
+  - `threads`: The number of threads to use (The default is 8 if unspecified)
+  - `n_predict`: The number of tokens to return (The default is 128 if unspecified)
+  - `seed`: The seed. The default is -1 (none)
+  - `top_k`
+  - `top_p`
+  - `temp`: temperature
+  - `batch_size`: batch size
+- `callback`: the streaming callback function that gets called every time the client gets any token response back from the model
+
+### Examples
+
+#### 1. Node.js
+
+Using node.js, you just need to initialize a Dalai object with `new Dalai()` and then use it.
+
+```javascript
+const Dalai = require('dalai')
+new Dalai().request({
+  model: "7B",
+  prompt: "The following is a conversation between a boy and a girl:",
+}, (token) => {
+  process.stdout.write(token)
+})
+```
+
+#### 2. Non node.js (socket.io)
+
+To make use of this in a browser or any other language, you can use thie socket.io API.
+
+##### Step 1. start a server
+
+First you need to run a Dalai socket server:
+
+```javascript
+// server.js
+const Dalai = require('dalai')
+new Dalai().serve(3000)     // port 3000
+```
+
+##### Step 2. connect to the server
+
+Then once the server is running, simply make requests to it by passing the `ws://localhost:3000` socket url when initializing the Dalai object:
+
+```javascript
+const Dalai = require("dalai")
+new Dalai("ws://localhost:3000").request({
+  model: "7B",
+  prompt: "The following is a conversation between a boy and a girl:",
+}, (token) => {
+  console.log("token", token)
+})
+```
+
+---
+
+## 3. serve()
+
+### Syntax
+
+Starts a socket.io server at `port`
+
+```javascript
+dalai.serve(port)
+```
+
+### Examples
+
+```javascript
+const Dalai = require("dalai")
+new Dalai().serve(3000)
+```
+
+---
+
+## 4. http()
+
+### Syntax
+
+connect with an existing `http` instance (The `http` npm package)
+
+```javascript
+dalai.http(http)
+```
+
+- `http`: The [http](https://nodejs.org/api/http.html) object
+
+### Examples
+
+This is useful when you're trying to plug dalai into an existing node.js web app
+
+```javascript
+const app = require('express')();
+const http = require('http').Server(app);
+dalai.http(http)
+http.listen(3000, () => {
+  console.log("server started")
+})
+```
--- a/docs/dalai.gif
+++ b/docs/dalai.gif
--- a/docs/index.html
+++ b/docs/index.html
@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>dalai</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="twitter:card" content="summary_large_image" />
+  <meta name="twitter:title" content="Dalai" />
+  <meta name="twitter:description" content="Dead simple way to run LLaMA on your computer" />
+  <meta name="twitter:image" content="https://cocktailpeanut.github.io/dalai/preview.png" />
+  <meta property="og:url" content="https://github.com/cocktailpeanut/dalai" />
+  <meta property="og:type" content="website" />
+  <meta property="og:title" content="Dalai" />
+  <meta property="og:description" content="IBrowse, Search, and Manage Stablediffusion Images in One Place" />
+  <meta property="og:image" content="https://cocktailpeanut.github.io/dalai/preview.png" />
+  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
+  <meta name="description" content="Dead simple way to run LLaMA on your computer">
+  <meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
+  <link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta2/css/all.min.css" rel="stylesheet">
+  <link rel="stylesheet" href="vue.css">
+  <link rel="stylesheet" href="style.css">
+</head>
+<body>
+  <div id="app"></div>
+  <script>
+    window.$docsify = {
+      name: '',
+      repo: '',
+      maxLevel: 3,
+  //    coverpage: true,
+      themeColor: "black",
+    }
+    document.addEventListener("DOMContentLoaded", () => {
+      setTimeout(() => {
+        let el = document.querySelector("#" + location.hash.split("=")[1])
+        if (el) {
+          const y = el.getBoundingClientRect().top + window.pageYOffset;
+          window.scrollTo({top: y});
+        }
+      }, 1000)
+
+      document.body.addEventListener("click", (e) => {
+        let el = e.target.closest("a")
+        if (el && el.getAttribute("href").startsWith("#/?id=")) {
+          location.hash = el.getAttribute("href").slice(1)
+        } else if (e.target.tagName === "A" && e.target.getAttribute("href").startsWith("#/?id=")) {
+          location.hash = e.target.getAttribute("href").slice(1)
+        }
+      })
+    })
+  </script>
+  <script src="//cdn.jsdelivr.net/npm/docsify/lib/docsify.min.js"></script>
+</body>
+</html>
--- a/docs/preview.png
+++ b/docs/preview.png
--- a/docs/style.css
+++ b/docs/style.css
@ -0,0 +1,218 @@
+:root {
+  --theme-color: rgb(154, 205, 50);
+  --theme-color-light: rgba(154, 205, 50, 0.05);
+  --theme-color-medium: rgba(154, 205, 50, 0.1);
+  --dark-color: rgba(0,0,0,0.8);
+  --dark-focused: rgb(154, 205, 50);
+
+}
+* {
+  word-wrap: break-word;
+}
+body {
+  font-family: "Helvetica Neue", helvetica, arial, "sans serif";
+  margin: 0;
+}
+section.cover .cover-main {
+  margin: 0 auto;
+  box-sizing: border-box;
+  padding-top: 50px;
+  height: 100%;
+  max-width: 1000px;
+  text-align: center;
+}
+.cover-main blockquote {
+  font-size: 14px;
+  margin: 0;
+  text-transform: uppercase;
+  font-family: Sans-serif !important;
+}
+.cover-main blockquote {
+  margin: 0 0 10px;
+  text-transform: uppercase;
+  font-family: Sans-serif !important;
+  font-size: 14px !important;
+}
+section.cover h1 img {
+  vertical-align: text-bottom;
+  height: 45px;
+  width: 45px;
+}
+section.cover h1 {
+  margin: 0 !important;
+  color: black;
+}
+section.cover p {
+  font-size: 14px;
+  margin: 0;
+}
+section.cover p a {
+  background: black;
+  color: white;
+  text-decoration: none;
+  display: inline-block;
+  padding: 5px 10px;
+  border-radius: 2px;
+}
+/*
+section.cover p {
+  margin: 0;
+  font-size: 14px;
+}
+*/
+section.cover .cover-main>p:last-child a {
+  margin: 20px 0 !important;
+}
+
+.sidebar {
+  border: none;
+  color: rgba(0,0,0,0.8);
+  background: rgba(0,0,100,0.05);
+}
+.sidebar li {
+  margin: 0;
+}
+.sidebar ul li a {
+  color: rgba(0,0,0,0.8);
+  line-height: 19px;
+  font-size: 13px;
+}
+.sidebar ul li.active>a {
+  color: var(--theme-color);
+  border: none !important;
+}
+.markdown-section {
+  max-width: 1000px;
+  margin: 20px;
+}
+.markdown-section blockquote {
+  padding: 5px 20px;
+  border: none !important;
+  background: rgba(0,0,100,0.05);
+}
+.cover-main h1 {
+  font-size: 40px !important;
+  letter-spacing: -2px;
+  font-weight: bold !important;
+}
+.cover-main img {
+  width: 50px;
+}
+section.cover .cover-main>p:last-child a, section.cover .btn {
+  padding: 2px 10px;
+  font-weight: bold;
+  margin: 5px 0;
+  border-radius: 2px;
+  background: black !important;
+  color: white;
+  border: none;
+  display: inline-block;
+}
+.markdown-section code, .markdown-section pre {
+  font-size: 12px;
+  line-height: 16px;
+}
+h1, h2, h3, h4, h5 {
+}
+.content {
+  padding-top: 0;
+}
+.markdown-section h1 {
+  font-size: 60px;
+  letter-spacing: -4px;
+}
+.markdown-section h2 {
+  font-size: 40px;
+  letter-spacing: -2px;
+}
+.markdown-section h3 {
+  font-size: 30px;
+  /*
+  border-left: 10px solid black;
+  padding-left: 15px;
+  */
+}
+.anchor span {
+  color: var(--dark-color);
+}
+.markdown-section ol, .markdown-section p, .markdown-section ul {
+  line-height: 1.4rem;
+}
+.markdown-section textarea {
+  width: 100%;
+  box-sizing: border-box;
+  padding: 20px;
+  height: 400px;
+  font-size: 12px;
+  border: none;
+  background: rgba(0,0,0,0.8);
+  color: white;
+}
+.home {
+  max-width: 800px;
+  margin: 0 auto;
+  padding: 100px 0;
+  text-align: center;
+  color: rgba(255,255,255,0.9);
+  background: rgba(0,0,0,0.9);
+}
+.home .item {
+  font-size: 12px;
+  font-weight: bold;
+  color: rgba(255,255,255,0.8);
+  padding: 10px 20px;
+  background: rgba(0,0,0,0.2);
+  text-decoration: none;
+  border-radius: 2px;
+  display: inline-block;
+}
+.home #title i {
+  font-size: 80px;
+}
+.home #title {
+  font-size: 35px;
+  letter-spacing: -1px;
+}
+a.btn {
+  display: inline-block;
+  background: var(--theme-color);
+  color: white;
+  padding: 5px 20px;
+  border-radius: 5px;
+  color: white;
+  text-decoration: none;
+  border: 2px solid black;
+}
+.btn.inverse {
+  border: 2px solid black;
+  color: black;
+  background: white;
+}
+section.cover.show {
+  background: rgba(0,0,100,0.05) !important;
+}
+.videoWrapper {
+  position: relative;
+  padding-bottom: 56.25%; /* 16:9 */
+  height: 0;
+  margin-top: 10px;
+}
+.videoWrapper iframe {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+}
+@media screen and (max-width: 840px){
+  a.btn {
+    display: block;
+    margin-bottom: 10px;
+    text-align: center;
+    width: 150px;
+    padding: 5px;
+  }
+  .markdown-section {
+    margin: 0;
+  }
+}
--- a/docs/vue.css
+++ b/docs/vue.css
--- a/index.js
+++ b/index.js
@ -0,0 +1,209 @@
+const os = require('os');
+const pty = require('node-pty');
+const path = require('path');
+const fs = require("fs");
+const { createServer } = require("http");
+const { Server } = require("socket.io");
+const { io } = require("socket.io-client");
+const term = require( 'terminal-kit' ).terminal;
+const Downloader = require("nodejs-file-downloader");
+const shell = os.platform() === 'win32' ? 'powershell.exe' : 'bash';
+class Dalai {
+  constructor(url) {
+    if (url) this.url = url
+    this.home = path.resolve(os.homedir(), "dalai")
+    try {
+      fs.mkdirSync(this.home, { recursive: true })
+    } catch (e) { }
+    this.config = {
+      name: 'xterm-color',
+      cols: 80,
+      rows: 30,
+    }
+  }
+  async download(model) {
+    const num = {
+      "7B": 1,
+      "13B": 2,
+      "30B": 4,
+      "65B": 8,
+    }
+    const files = ["checklist.chk", "params.json"]
+    for(let i=0; i<num[model]; i++) {
+      files.push(`consolidated.0${i}.pth`)
+    }
+    const resolvedPath = path.resolve(this.home, "models", model)
+    await fs.promises.mkdir(resolvedPath, { recursive: true }).catch((e) => { })
+
+    for(let file of files) {
+      const task = `downloading ${file}`
+      const downloader = new Downloader({
+        url: `https://agi.gpt4.org/llama/LLaMA/${model}/${file}`,
+        directory: path.resolve(this.home, "models", model),
+        onProgress: (percentage, chunk, remainingSize) => {
+          this.progress(task, percentage)
+        },
+      });
+      try {
+        await this.startProgress(task)
+        await downloader.download();
+      } catch (error) {
+        console.log(error);
+      }
+      this.progressBar.update(1);
+      term("\n")
+    }
+
+    const files2 = ["tokenizer_checklist.chk", "tokenizer.model"]
+    for(let file of files2) {
+      const task = `downloading ${file}`
+      const downloader = new Downloader({
+        url: `https://agi.gpt4.org/llama/LLaMA/${file}`,
+        directory: path.resolve(this.home, "models"),
+        onProgress: (percentage, chunk, remainingSize) => {
+          this.progress(task, percentage)
+        },
+      });
+      try {
+        await this.startProgress(task)
+        await downloader.download();
+      } catch (error) {
+        console.log(error);
+      }
+      this.progressBar.update(1);
+      term("\n")
+    }
+
+  }
+  async install(...models) {
+    // install to ~/llama.cpp
+    await this.exec("git clone https://github.com/ggerganov/llama.cpp.git dalai", os.homedir())
+    await this.exec("make", this.home)
+    for(let model of models) {
+      await this.download(model)
+      await this.exec(`python3 convert-pth-to-ggml.py models/${model}/ 1`, this.home)
+      await this.quantize(model)
+    }
+  }
+  serve(port) {
+    const httpServer = createServer();
+    const io = new Server(httpServer)
+    io.on("connection", (socket) => {
+      socket.on('request', async (req) => {
+        await this.query(req, (str) => {
+          io.emit("result", { response: str, request: req })
+        })
+      });
+    });
+    httpServer.listen(port)
+  }
+  http(httpServer) {
+    const io = new Server(httpServer)
+    io.on("connection", (socket) => {
+      socket.on('request', async (req) => {
+        await this.query(req, (str) => {
+          io.emit("result", { response: str, request: req })
+        })
+      });
+    });
+  }
+  async request(req, cb) {
+    if (this.url) {
+      await this.connect(req, cb)
+    } else {
+      await this.query(req, cb)
+    }
+  }
+  async query(req, cb) {
+    let o = {
+      seed: req.seed || -1,
+      threads: req.threads || 8,
+      n_predict: req.n_predict || 128,
+      model: `./models/${req.model || "7B"}/ggml-model-q4_0.bin`
+    }
+    if (req.top_k) o.top_k = req.top_k
+    if (req.top_p) o.top_p = req.top_p
+    if (req.temp) o.temp = req.temp
+    if (req.batch_size) o.batch_size = req.batch_size
+
+    let chunks = [] 
+    for(let key in o) {
+      chunks.push(`--${key} ${o[key]}`) 
+    }
+    chunks.push(`-p "${req.prompt}"`)
+
+    if (req.full) {
+      await this.exec(`./main ${chunks.join(" ")}`, this.home, cb)
+    } else {
+      const startpattern = /.*sampling parameters:.*/g
+      const endpattern = /.*mem per token.*/g
+      let started = false
+      let ended = false
+      await this.exec(`./main ${chunks.join(" ")}`, this.home, (msg) => {
+        if (endpattern.test(msg)) ended = true
+        if (started && !ended) {
+          cb(msg)
+        }
+        if (startpattern.test(msg)) started = true
+      })
+    }
+  }
+  connect(req, cb) {
+    const socket = io(this.url)
+    socket.emit('request', req)
+    socket.on('response', cb)
+    socket.on('error', function(e) {
+      throw e
+    });
+  }
+  exec(cmd, cwd, cb) {
+    return new Promise((resolve, reject) => {
+      const config = Object.assign({}, this.config)
+      if (cwd) {
+        config.cwd = path.resolve(cwd)
+      }
+      const ptyProcess = pty.spawn(shell, [], config)
+      ptyProcess.onData((data) => {
+        if (cb) {
+          cb(data)
+        } else {
+          process.stdout.write(data);
+        }
+      });
+      ptyProcess.onExit((res) => {
+        resolve(res)
+      });
+      ptyProcess.write(`${cmd}\r`)
+      ptyProcess.write("exit\r")
+    })
+  }
+  async quantize(model) {
+    let num = {
+      "7B": 1,
+      "13B": 2,
+      "30B": 4,
+      "65B": 8,
+    }
+    for(let i=0; i<num[model]; i++) {
+      const suffix = (i === 0 ? "" : `.${i}`)
+      await this.exec(`./quantize ./models/${model}/ggml-model-f16.bin ./models/${model}/ggml-model-q4_0.bin${suffix} 2`, this.home)
+    }
+  }
+  progress(task, percent) {
+    this.progressBar.update(percent/100);
+    //if (percent >= 100) {
+    //  setTimeout(() => {
+    //    term("\n")
+    //  }, 200)
+    //}
+  }
+  startProgress(title) {
+    this.progressBar = term.progressBar({
+      width: 120,
+      title,
+      eta: true ,
+      percent: true
+    });
+  }
+}
+module.exports = Dalai
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@ -0,0 +1,18 @@
+{
+  "name": "dalai",
+  "version": "0.0.7",
+  "description": "",
+  "main": "index.js",
+  "author": "cocktailpeanut",
+  "license": "MIT",
+  "dependencies": {
+    "node-pty": "^0.10.1",
+    "nodejs-file-downloader": "^4.10.6",
+    "socket.io": "^4.6.1",
+    "socket.io-client": "^4.6.1",
+    "terminal-kit": "^3.0.0",
+    "ejs": "^3.1.8",
+    "express": "^4.18.2"
+  },
+  "bin": "bin/cli.js"
+}