diff --git a/tools/server/chat-llama2.sh b/tools/server/chat-llama2.sh
deleted file mode 100755
index 450445f17..000000000
--- a/tools/server/chat-llama2.sh
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env bash
-
-API_URL="${API_URL:-http://127.0.0.1:8080}"
-
-CHAT=(
- "Hello, Assistant."
- "Hello. How may I help you today?"
-)
-
-INSTRUCTION="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."
-
-trim() {
- shopt -s extglob
- set -- "${1##+([[:space:]])}"
- printf "%s" "${1%%+([[:space:]])}"
-}
-
-trim_trailing() {
- shopt -s extglob
- printf "%s" "${1%%+([[:space:]])}"
-}
-
-format_prompt() {
- if [[ "${#CHAT[@]}" -eq 0 ]]; then
- echo -n "[INST] <>\n${INSTRUCTION}\n<>"
- else
- LAST_INDEX=$(( ${#CHAT[@]} - 1 ))
- echo -n "${CHAT[$LAST_INDEX]}\n[INST] $1 [/INST]"
- fi
-}
-
-tokenize() {
- curl \
- --silent \
- --request POST \
- --url "${API_URL}/tokenize" \
- --header "Content-Type: application/json" \
- --data-raw "$(jq -ns --arg content "$1" '{content:$content}')" \
- | jq '.tokens[]'
-}
-
-N_KEEP=$(tokenize "[INST] <>\n${INSTRUCTION}\n<>" | wc -l)
-
-chat_completion() {
- PROMPT="$(trim_trailing "$(format_prompt "$1")")"
- DATA="$(echo -n "$PROMPT" | jq -Rs --argjson n_keep $N_KEEP '{
- prompt: .,
- temperature: 0.2,
- top_k: 40,
- top_p: 0.9,
- n_keep: $n_keep,
- n_predict: 1024,
- stop: ["[INST]"],
- stream: true
- }')"
-
- # Create a temporary file to hold the Python output
- TEMPFILE=$(mktemp)
-
- exec 3< <(curl \
- --silent \
- --no-buffer \
- --request POST \
- --url "${API_URL}/completion" \
- --header "Content-Type: application/json" \
- --data-raw "${DATA}")
-
- python -c "
-import json
-import sys
-
-answer = ''
-while True:
- line = sys.stdin.readline()
- if not line:
- break
- if line.startswith('data: '):
- json_content = line[6:].strip()
- content = json.loads(json_content)['content']
- sys.stdout.write(content)
- sys.stdout.flush()
- answer += content
-
-answer = answer.rstrip('\n')
-
-# Write the answer to the temporary file
-with open('$TEMPFILE', 'w') as f:
- f.write(answer)
- " <&3
-
- exec 3<&-
-
- # Read the answer from the temporary file
- ANSWER=$(cat $TEMPFILE)
-
- # Clean up the temporary file
- rm $TEMPFILE
-
- printf "\n"
-
- CHAT+=("$1" "$(trim "$ANSWER")")
-}
-
-while true; do
- echo -en "\033[0;32m" # Green color
- read -r -e -p "> " QUESTION
- echo -en "\033[0m" # Reset color
- chat_completion "${QUESTION}"
-done
diff --git a/tools/server/chat.mjs b/tools/server/chat.mjs
deleted file mode 100644
index 4fef5655a..000000000
--- a/tools/server/chat.mjs
+++ /dev/null
@@ -1,131 +0,0 @@
-import * as readline from 'node:readline'
-import { stdin, stdout } from 'node:process'
-import { readFileSync } from 'node:fs'
-import { SchemaConverter } from './public_legacy/json-schema-to-grammar.mjs'
-
-const args = process.argv.slice(2);
-const grammarJsonSchemaFile = args.find(
- (_, index) => args[index - 1] === "--grammar-json-schema"
-);
-
-const no_cached_prompt = args.find(
- (_, index) => args[index - 1] === "--no-cache-prompt"
-) ?? "false";
-
-const grammarFile = args.find((_, index) => args[index - 1] === "--grammar");
-
-// Example usage: function,arguments
-const grammarJsonSchemaPropOrder = args.find(
- (_, index) => args[index - 1] === "--grammar-json-schema-prop-order"
-);
-const propOrder = grammarJsonSchemaPropOrder
- ? grammarJsonSchemaPropOrder
- .split(",")
- .reduce((acc, cur, index) => ({ ...acc, [cur]: index }), {})
- : {};
-
-let grammar = null
-if (grammarJsonSchemaFile) {
- let schema = JSON.parse(readFileSync(grammarJsonSchemaFile, 'utf-8'))
- const converter = new SchemaConverter({prop_order: propOrder, allow_fetch: true})
- schema = await converter.resolveRefs(schema, grammarJsonSchemaFile)
- converter.visit(schema, '')
- grammar = converter.formatGrammar()
-}
-if (grammarFile) {
- grammar = readFileSync(grammarFile, 'utf-8')
-}
-
-// for cached prompt
-let slot_id = -1;
-
-const API_URL = 'http://127.0.0.1:8080'
-
-const chat = [
- {
- human: "Hello, Assistant.",
- assistant: "Hello. How may I help you today?"
- },
- {
- human: "Please tell me the largest city in Europe.",
- assistant: "Sure. The largest city in Europe is Moscow, the capital of Russia."
- },
-]
-
-const instruction = `A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.`
-
-function format_prompt(question) {
- return `${instruction}\n${
- chat.map(m =>`### Human: ${m.human}\n### Assistant: ${m.assistant}`).join("\n")
- }\n### Human: ${question}\n### Assistant:`
-}
-
-async function tokenize(content) {
- const result = await fetch(`${API_URL}/tokenize`, {
- method: 'POST',
- body: JSON.stringify({ content })
- })
-
- if (!result.ok) {
- return []
- }
-
- return await result.json().tokens
-}
-
-const n_keep = await tokenize(instruction).length
-
-async function chat_completion(question) {
- const result = await fetch(`${API_URL}/completion`, {
- method: 'POST',
- body: JSON.stringify({
- prompt: format_prompt(question),
- temperature: 0.2,
- top_k: 40,
- top_p: 0.9,
- n_keep: n_keep,
- n_predict: 256,
- cache_prompt: no_cached_prompt === "false",
- slot_id: slot_id,
- stop: ["\n### Human:"], // stop completion after generating this
- grammar,
- stream: true,
- })
- })
-
- if (!result.ok) {
- return
- }
-
- let answer = ''
-
- for await (var chunk of result.body) {
- const t = Buffer.from(chunk).toString('utf8')
- if (t.startsWith('data: ')) {
- const message = JSON.parse(t.substring(6))
- slot_id = message.slot_id
- answer += message.content
- process.stdout.write(message.content)
- if (message.stop) {
- if (message.truncated) {
- chat.shift()
- }
- break
- }
- }
- }
-
- process.stdout.write('\n')
- chat.push({ human: question, assistant: answer.trimStart() })
-}
-
-const rl = readline.createInterface({ input: stdin, output: stdout });
-
-const readlineQuestion = (rl, query, options) => new Promise((resolve, reject) => {
- rl.question(query, options, resolve)
-});
-
-while(true) {
- const question = await readlineQuestion(rl, '> ')
- await chat_completion(question)
-}
diff --git a/tools/server/chat.sh b/tools/server/chat.sh
deleted file mode 100755
index 84cea2d56..000000000
--- a/tools/server/chat.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-
-API_URL="${API_URL:-http://127.0.0.1:8080}"
-
-CHAT=(
- "Hello, Assistant."
- "Hello. How may I help you today?"
- "Please tell me the largest city in Europe."
- "Sure. The largest city in Europe is Moscow, the capital of Russia."
-)
-
-INSTRUCTION="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."
-
-trim() {
- shopt -s extglob
- set -- "${1##+([[:space:]])}"
- printf "%s" "${1%%+([[:space:]])}"
-}
-
-trim_trailing() {
- shopt -s extglob
- printf "%s" "${1%%+([[:space:]])}"
-}
-
-format_prompt() {
- echo -n "${INSTRUCTION}"
- printf "\n### Human: %s\n### Assistant: %s" "${CHAT[@]}" "$1"
-}
-
-tokenize() {
- curl \
- --silent \
- --request POST \
- --url "${API_URL}/tokenize" \
- --header "Content-Type: application/json" \
- --data-raw "$(jq -ns --arg content "$1" '{content:$content}')" \
- | jq '.tokens[]'
-}
-
-N_KEEP=$(tokenize "${INSTRUCTION}" | wc -l)
-
-chat_completion() {
- PROMPT="$(trim_trailing "$(format_prompt "$1")")"
- DATA="$(echo -n "$PROMPT" | jq -Rs --argjson n_keep $N_KEEP '{
- prompt: .,
- temperature: 0.2,
- top_k: 40,
- top_p: 0.9,
- n_keep: $n_keep,
- n_predict: 256,
- cache_prompt: true,
- stop: ["\n### Human:"],
- stream: true
- }')"
-
- ANSWER=''
-
- while IFS= read -r LINE; do
- if [[ $LINE = data:* ]]; then
- CONTENT="$(echo "${LINE:5}" | jq -r '.content')"
- printf "%s" "${CONTENT}"
- ANSWER+="${CONTENT}"
- fi
- done < <(curl \
- --silent \
- --no-buffer \
- --request POST \
- --url "${API_URL}/completion" \
- --header "Content-Type: application/json" \
- --data-raw "${DATA}")
-
- printf "\n"
-
- CHAT+=("$1" "$(trim "$ANSWER")")
-}
-
-while true; do
- read -r -e -p "> " QUESTION
- chat_completion "${QUESTION}"
-done