Improved sync logic, show more statistics, bug fixes

discord/.gitignore

@@ -0,0 +1 @@
+db.sqlite

discord/bot.ts

@@ -4,7 +4,7 @@
  */
 
 import { Client, Events, GatewayIntentBits, MessageReaction, PartialMessageReaction, Partials, User } from 'discord.js';
-import { db, openDb, reactionEmojis, recordReaction } from './util';
+import { db, openDb, reactionEmojis, recordReaction, sync } from './util';
 
 const client = new Client({
     intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildMessages, GatewayIntentBits.GuildMessageReactions],
@@ -57,6 +57,7 @@ async function startup() {
     console.log("[db] Ready.");
     console.log("[bot] Logging in...");
     await client.login(process.env.TOKEN);
+    await sync(client.guilds);
 }
 
 startup();

discord/sync.ts

@@ -1,20 +1,11 @@
 /**
  * sync.ts
- * Syncs the message reactions in chat with the database, for when the bot is not running.
+ * Syncs message reactions in chat with the database.
+ * The bot will do the same thing automatically upon startup, but this is just in the form of a standalone script.
  */
 
-import {
-    Client,
-    Collection,
-    Events,
-    GatewayIntentBits,
-    GuildTextBasedChannel,
-    IntentsBitField,
-    Message,
-    MessageReaction,
-    Partials
-} from 'discord.js';
-import { db, clearDb, openDb, reactionEmojis, recordReaction } from './util';
+import { Client, Events, GatewayIntentBits, IntentsBitField, Partials } from 'discord.js';
+import { db, openDb, reactionEmojis, sync } from './util';
 
 const client = new Client({
     intents: [GatewayIntentBits.MessageContent, IntentsBitField.Flags.Guilds, IntentsBitField.Flags.GuildMessages],
@@ -28,8 +19,6 @@ client.once(Events.ClientReady, async () => {
 });
 
 async function startup() {
-    console.log("[db] Clearing database...");
-    clearDb();
     console.log("[db] Opening...");
     await openDb();
     console.log("[db] Migrating...");
@@ -37,39 +26,7 @@ async function startup() {
     console.log("[db] Ready.");
     console.log("[bot] Logging in...");
     await client.login(process.env.TOKEN);
-
-    const guild = await client.guilds.fetch(process.env.GUILD);
-    if (!guild) {
-        console.error(`[bot] FATAL: guild ${guild.id} not found!`);
-        return 1;
-    }
-    console.log(`[bot] Entered guild ${guild.id}`);
-    const channels = await guild.channels.fetch();
-    const textChannels = <Collection<string, GuildTextBasedChannel>> channels.filter(c => c && 'messages' in c && c.isTextBased);
-    for (const [id, textChannel] of textChannels) {
-        console.log(`[bot] Found text channel ${id}`);
-        let before: string = undefined;
-        let messages = new Collection<string, Message<true>>();
-        let newMessages: Collection<string, Message<true>>;
-        try {
-            do {
-                newMessages = await textChannel.messages.fetch({before, limit: 100});
-                messages = messages.concat(newMessages);
-                console.log(`[bot] [${id}] Fetched ${messages.size} messages (+${newMessages.size})`);
-                if (messages.size > 0)
-                    before = messages.last().id;
-            } while (newMessages.size > 0);
-            console.log(`[bot] [${id}] Fetched all messages.`);
-            const reactions = messages.flatMap<MessageReaction>(m => m.reactions.cache);
-            console.log(`[bot] Found ${reactions.size} reactions`);
-            for (const [_, reaction] of reactions) {
-                await recordReaction(reaction);
-            }
-            console.log(`[bot] [${id}] Finished recording reactions.`);
-        } catch (err) {
-            console.warn(`[bot] [${id}] Failed to fetch messages and reactions: ${err}`);
-        }
-    }
+    await sync(client.guilds);
     process.exit(0);
 }
 

discord/util.ts

@@ -3,7 +3,7 @@
  * Common helper functions
  */
 
-import { MessageReaction, User } from 'discord.js';
+import { Collection, GuildManager, GuildTextBasedChannel, Message, MessageReaction, User } from 'discord.js';
 import { createWriteStream, existsSync, unlinkSync } from 'fs';
 import { get as httpGet } from 'https';
 import { Database, open } from 'sqlite';
@@ -41,7 +41,7 @@ async function downloadUserAvatar(user: User)
     console.log(`[bot] Downloading ${user.id}'s avatar...`);
     const file = createWriteStream(userAvatarPath(user));
     return new Promise<void>(resolve => {
-        httpGet(user.avatarURL(), res => {
+        httpGet(user.displayAvatarURL(), res => {
             res.pipe(file);
             file.on('finish', () => {
                 file.close();
@@ -83,7 +83,7 @@ async function recordReaction(reaction: MessageReaction)
     }
     try {
         await db.run(
-            `INSERT INTO messages(id, guild, channel, author, content, reaction_${emojiIdx}_count) VALUES(?, ?, ?, ?, ?, 1)
+            `INSERT INTO messages(id, guild, channel, author, content, reaction_${emojiIdx}_count) VALUES(?, ?, ?, ?, ?, ?)
             ON CONFLICT(id) DO UPDATE SET reaction_${emojiIdx}_count = ? WHERE id = ?`,
             reaction.message.id,
             reaction.message.guildId,
@@ -91,6 +91,7 @@ async function recordReaction(reaction: MessageReaction)
             reaction.message.author.id,
             reaction.message.content,
             reaction.count,
+            reaction.count,
             reaction.message.id
         );
         await refreshUserReactionTotalCount(reaction.message.author, emojiIdx);
@@ -101,4 +102,62 @@ async function recordReaction(reaction: MessageReaction)
     }
 }
 
-export { db, clearDb, openDb, reactionEmojis, recordReaction };
+async function sync(guilds: GuildManager) {
+    const guild = await guilds.fetch(process.env.GUILD);
+    if (!guild) {
+        console.error(`[bot] FATAL: guild ${guild.id} not found!`);
+        return 1;
+    }
+    console.log(`[bot] Entered guild ${guild.id}`);
+    const channels = await guild.channels.fetch();
+    const textChannels = <Collection<string, GuildTextBasedChannel>> channels.filter(c => c && 'messages' in c && c.isTextBased);
+    for (const [id, textChannel] of textChannels) {
+        console.log(`[bot] Found text channel ${id}`);
+        const oldestMsg = await db.get<ScoreboardMessageRow>(
+            'SELECT * FROM messages WHERE guild = ? AND channel = ? ORDER BY id ASC LIMIT 1',
+            guild.id,
+            id
+        );
+        const newestMsg = await db.get<ScoreboardMessageRow>(
+            'SELECT * FROM messages WHERE guild = ? AND channel = ? ORDER BY id DESC LIMIT 1',
+            guild.id,
+            id
+        );
+        let before: string = oldestMsg && String(oldestMsg.id);
+        let after: string = newestMsg && String(newestMsg.id);
+        let messagesCount = 0;
+        let reactionsCount = 0;
+        let newMessagesBefore: Collection<string, Message<true>>;
+        let newMessagesAfter: Collection<string, Message<true>>;
+        try {
+            do {
+                newMessagesBefore = await textChannel.messages.fetch({ before, limit: 100 });
+                messagesCount += newMessagesBefore.size;
+
+                newMessagesAfter = await textChannel.messages.fetch({ after, limit: 100 });
+                messagesCount += newMessagesAfter.size;
+                console.log(`[bot] [${id}] Fetched ${messagesCount} messages (+${newMessagesBefore.size} older, ${newMessagesAfter.size} newer)`);
+
+                const reactions = newMessagesBefore.flatMap<MessageReaction>(m => m.reactions.cache)
+                    .concat(newMessagesAfter.flatMap<MessageReaction>(m => m.reactions.cache));
+                for (const [_, reaction] of reactions) {
+                    await recordReaction(reaction);
+                }
+                reactionsCount += reactions.size;
+                console.log(`[bot] [${id}] Recorded ${reactionsCount} reactions (+${reactions.size}).`);
+
+                if (newMessagesBefore.size > 0) {
+                    before = newMessagesBefore.last().id;
+                }
+                if (newMessagesAfter.size > 0) {
+                    after = newMessagesAfter.first().id;
+                }
+            } while (newMessagesBefore.size === 100 || newMessagesAfter.size === 100);
+            console.log(`[bot] [${id}] Done.`);
+        } catch (err) {
+            console.warn(`[bot] [${id}] Failed to fetch messages and reactions: ${err}`);
+        }
+    }
+}
+
+export { db, clearDb, openDb, reactionEmojis, recordReaction, sync };

models.ts

@@ -5,9 +5,9 @@
 
 interface ScoreboardMessageRow {
     id: number,
-    author: number,
     guild: number,
     channel: number,
+    author: string,
     content: string,
     reaction_1_count: number,
     reaction_2_count: number,
@@ -15,7 +15,7 @@ interface ScoreboardMessageRow {
 }
 
 interface ScoreboardUserRow {
-    id: number,
+    id: string,
     username: string,
     reaction_1_total: number,
     reaction_2_total: number,

server.ts

@@ -19,17 +19,20 @@ async function openDb() {
     return open({
         filename: 'discord/db.sqlite',
         driver: Database3
-    })
+    });
 }
 
 app.get('/', async (req, res) => {
-    const users = await db.all<[ScoreboardUserRow]>('SELECT * FROM users');
-    //const msgs = await db.all<[ScoreboardMessageRow]>('SELECT * FROM messages');
+    const msg1 = await db.all<[ScoreboardMessageRow]>('SELECT * FROM messages ORDER BY reaction_1_count DESC LIMIT 5');
+    const msg2 = await db.all<[ScoreboardMessageRow]>('SELECT * FROM messages ORDER BY reaction_2_count DESC LIMIT 5');
+    const msg3 = await db.all<[ScoreboardMessageRow]>('SELECT * FROM messages ORDER BY reaction_3_count DESC LIMIT 5');
+    const bestMsg = await db.all<[ScoreboardMessageRow]>('SELECT *, SUM(reaction_1_count)+SUM(reaction_2_count)+SUM(reaction_3_count) AS all_reacts FROM messages GROUP BY id ORDER BY all_reacts DESC LIMIT 5');
 
-    const funniest = [...users].sort((a, b) => a.reaction_1_total - b.reaction_1_total);
-    const realest = [...users].sort((a, b) => a.reaction_2_total - b.reaction_2_total);
-    const cunniest = [...users].sort((a, b) => a.reaction_3_total - b.reaction_3_total);
-    res.render('index', { funniest, realest, cunniest });
+    const funniest = await db.all<[ScoreboardUserRow]>('SELECT * FROM users ORDER BY reaction_1_total DESC');
+    const realest = await db.all<[ScoreboardUserRow]>('SELECT * FROM users ORDER BY reaction_2_total DESC');
+    const cunniest = await db.all<[ScoreboardUserRow]>('SELECT * FROM users ORDER BY reaction_3_total DESC');
+
+    res.render('index', { funniest, realest, cunniest, msg1, msg2, msg3, bestMsg });
 });
 
 app.listen(port, async () => {

views/index.pug

@@ -49,3 +49,84 @@ html
                             img(src="/avatars/" + row.id + ".webp", height="64")
                         td
                             span= row.reaction_3_total
+        br
+        h1 Funniest Messages 💀
+        table
+            tbody
+                each row in msg1
+                    tr
+                        td
+                            img(src="/avatars/" + row.author + ".webp", height="48")
+                        td
+                            if row.content
+                                span= row.content
+                            else
+                                span
+                                    i (media)
+                    tr
+                        td
+                            a(href="https://discord.com/channels/" + row.guild + "/" + row.channel + "/" + row.id) Link
+                        td
+                            span= "💀 " + row.reaction_1_count
+        br
+        h1 Realest Messages 💯
+        table
+            tbody
+                each row in msg2
+                    tr
+                        td
+                            img(src="/avatars/" + row.author + ".webp", height="48")
+                        td
+                            if row.content
+                                span= row.content
+                            else
+                                span
+                                    i (media)
+                    tr
+                        td
+                            a(href="https://discord.com/channels/" + row.guild + "/" + row.channel + "/" + row.id) Link
+                        td
+                            span= "💯 " + row.reaction_2_count
+        br
+        h1 Cunniest Messages 😭
+        table
+            tbody
+                each row in msg3
+                    tr
+                        td
+                            img(src="/avatars/" + row.author + ".webp", height="48")
+                        td
+                            if row.content
+                                span= row.content
+                            else
+                                span
+                                    i (media)
+                    tr
+                        td
+                            a(href="https://discord.com/channels/" + row.guild + "/" + row.channel + "/" + row.id) Link
+                        td
+                            span= "😭 " + row.reaction_3_count
+        br
+        h1 Best Messages 💀💯😭
+        table
+            tbody
+                each row in bestMsg
+                    tr
+                        td
+                            img(src="/avatars/" + row.author + ".webp", height="48")
+                        td
+                            if row.content
+                                span= row.content
+                            else
+                                span
+                                    i (media)
+                    tr
+                        td
+                            a(href="https://discord.com/channels/" + row.guild + "/" + row.channel + "/" + row.id) Link
+                        td
+                            if row.reaction_1_count
+                                span= " 💀 " + row.reaction_1_count
+                            if row.reaction_2_count
+                                span= " 💯 " + row.reaction_2_count
+                            if row.reaction_3_count
+                                span= " 😭 " + row.reaction_3_count