LLaMA test

discord/.env.example

@@ -2,6 +2,8 @@ TOKEN="sadkfl;jasdkl;fj"
 REACTIONS="💀,💯,😭"
 GUILD="123456789012345678"
 
+LLAMACPP_PORT=9999
+
 ENABLE_MOTD=1
 MOTD_CHANNEL="123456789012345678"
 MOTD_HREF="https://fembooru.jp/post/list"

discord/bot.ts

@@ -5,14 +5,17 @@
 
 import {
     Client,
+    Collection,
     Events,
     GatewayIntentBits,
+    Message,
     MessageReaction,
     PartialMessageReaction,
     Partials,
     TextChannel,
     User
 } from 'discord.js';
+import { ChatMessage, llamacpp, streamText } from 'modelfusion';
 import { logError, logInfo } from '../logging';
 import {
     db,
@@ -28,6 +31,13 @@ const client = new Client({
     partials: [Partials.Message, Partials.Channel, Partials.Reaction],
 });
 
+const llamaCppServer = llamacpp.Api({
+    baseUrl: {
+        host: "localhost",
+        port: process.env.LLAMACPP_PORT,
+    }
+});
+
 client.once(Events.ClientReady, async () => {
     logInfo('[bot] Ready.');
     for (let i = 0; i < reactionEmojis.length; ++i)
@@ -93,6 +103,47 @@ async function scheduleRandomMessage(firstTime = false)
     setTimeout(scheduleRandomMessage, timeoutMins * 60 * 1000);
 }
 
+async function llamaChat(history: Collection<string, Message>)
+{
+    const textStream = await streamText({
+        model: llamacpp
+            .CompletionTextGenerator({
+                promptTemplate: llamacpp.prompt.Llama2,
+                maxGenerationTokens: 400,
+                temperature: 0.7,
+                api: llamaCppServer
+            })
+            .withChatPrompt(),
+        prompt: {
+            system: "You are Hatsune Miku, the famous 16-year-old Japanese virtual singer from Crypton Future Media. You have a penchant for politically incorrect humor, and are making banter with your like-minded friends.",
+            messages: history.map<ChatMessage>(msg => ({
+                "role": msg.author.id === client.user.id ? "assistant" : "user",
+                "content": msg.content
+            }))
+        }
+    });
+
+    let outMsg: Message;
+    try {
+        for await (const textPart of textStream) {
+            if (!outMsg) {
+                outMsg = await history.first().channel.send(textPart);
+            } else {
+                await outMsg.edit(outMsg.content + textPart);
+            }
+        }
+    } catch (err) {
+        console.error(err);
+        await history.first().channel.send(err.toString());
+    }
+}
+client.on(Events.InteractionCreate, async interaction => {
+    if (!interaction.isChatInputCommand()) return;
+    console.log(interaction);
+    const history = await interaction.channel.messages.fetch({ limit: 5 });
+    await llamaChat(history);
+});
+
 client.on(Events.MessageReactionAdd, onMessageReactionChanged);
 client.on(Events.MessageReactionRemove, onMessageReactionChanged);
 

discord/package-lock.json

@@ -1,15 +1,16 @@
 {
-  "name": "femleaderboardbot",
+  "name": "femscoreboardbot",
   "version": "1.0.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "femleaderboardbot",
+      "name": "femscoreboardbot",
       "version": "1.0.0",
       "dependencies": {
         "discord.js": "^14.13.0",
         "dotenv": "^16.3.1",
+        "modelfusion": "^0.135.1",
         "sqlite": "^5.0.1",
         "sqlite3": "^5.1.6"
       },
@@ -281,6 +282,25 @@
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
     },
+    "node_modules/base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
     "node_modules/brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -463,6 +483,14 @@
       "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==",
       "optional": true
     },
+    "node_modules/eventsource-parser": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-1.1.1.tgz",
+      "integrity": "sha512-3Ej2iLj6ZnX+5CMxqyUb8syl9yVZwcwm8IIMrOJlF7I51zxOOrRlU3zxSb/6hFbl03ts1ZxHAGJdWLZOLyKG7w==",
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -650,6 +678,14 @@
       "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
       "optional": true
     },
+    "node_modules/js-tiktoken": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.7.tgz",
+      "integrity": "sha512-biba8u/clw7iesNEWLOLwrNGoBP2lA+hTaBLs/D45pJdUPFXyxD6nhcDVtADChghv4GgyAiMKYMiRx7x6h7Biw==",
+      "dependencies": {
+        "base64-js": "^1.5.1"
+      }
+    },
     "node_modules/lodash": {
       "version": "4.17.21",
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
@@ -835,11 +871,46 @@
         "node": ">=10"
       }
     },
+    "node_modules/modelfusion": {
+      "version": "0.135.1",
+      "resolved": "https://registry.npmjs.org/modelfusion/-/modelfusion-0.135.1.tgz",
+      "integrity": "sha512-Ds8xARmwPiXX2Hltu/ZVZait4v8RcXnpmkRPWAdWAng6SO/Gh0hqymnK3miQqLAsnysReWSNNcWTdO/94YMcHQ==",
+      "dependencies": {
+        "eventsource-parser": "1.1.1",
+        "js-tiktoken": "1.0.7",
+        "nanoid": "3.3.6",
+        "secure-json-parse": "2.7.0",
+        "type-fest": "4.9.0",
+        "ws": "8.14.2",
+        "zod": "3.22.4",
+        "zod-to-json-schema": "3.22.3"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
+    "node_modules/nanoid": {
+      "version": "3.3.6",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz",
+      "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
+        }
+      ],
+      "bin": {
+        "nanoid": "bin/nanoid.cjs"
+      },
+      "engines": {
+        "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
+      }
+    },
     "node_modules/negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -1088,6 +1159,11 @@
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
       "optional": true
     },
+    "node_modules/secure-json-parse": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
+      "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
+    },
     "node_modules/semver": {
       "version": "7.5.4",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
@@ -1268,6 +1344,17 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
       "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
     },
+    "node_modules/type-fest": {
+      "version": "4.9.0",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.9.0.tgz",
+      "integrity": "sha512-KS/6lh/ynPGiHD/LnAobrEFq3Ad4pBzOlJ1wAnJx9N4EYoqFhMfLIBjUT2UEx4wg5ZE+cC1ob6DCSpppVo+rtg==",
+      "engines": {
+        "node": ">=16"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/typescript": {
       "version": "5.2.2",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz",
@@ -1381,6 +1468,22 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+    },
+    "node_modules/zod": {
+      "version": "3.22.4",
+      "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz",
+      "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==",
+      "funding": {
+        "url": "https://github.com/sponsors/colinhacks"
+      }
+    },
+    "node_modules/zod-to-json-schema": {
+      "version": "3.22.3",
+      "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.22.3.tgz",
+      "integrity": "sha512-9isG8SqRe07p+Aio2ruBZmLm2Q6Sq4EqmXOiNpDxp+7f0LV6Q/LX65fs5Nn+FV/CzfF3NLBoksXbS2jNYIfpKw==",
+      "peerDependencies": {
+        "zod": "^3.22.4"
+      }
     }
   }
 }

discord/package.json

@@ -4,6 +4,7 @@
   "dependencies": {
     "discord.js": "^14.13.0",
     "dotenv": "^16.3.1",
+    "modelfusion": "^0.135.1",
     "sqlite": "^5.0.1",
     "sqlite3": "^5.1.6"
   },