From 657afe4755eff7558f581237fa73439db730e3d2 Mon Sep 17 00:00:00 2001 From: James Shiffer Date: Sun, 8 Oct 2023 13:48:41 -0700 Subject: [PATCH] Improved sync logic, show more statistics, bug fixes --- discord/.gitignore | 1 + discord/bot.ts | 3 +- discord/sync.ts | 53 +++--------------------------- discord/util.ts | 67 +++++++++++++++++++++++++++++++++++--- models.ts | 4 +-- server.ts | 17 ++++++---- views/index.pug | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 164 insertions(+), 62 deletions(-) create mode 100644 discord/.gitignore diff --git a/discord/.gitignore b/discord/.gitignore new file mode 100644 index 0000000..2fa69c2 --- /dev/null +++ b/discord/.gitignore @@ -0,0 +1 @@ +db.sqlite diff --git a/discord/bot.ts b/discord/bot.ts index 88268b7..73f85db 100644 --- a/discord/bot.ts +++ b/discord/bot.ts @@ -4,7 +4,7 @@ */ import { Client, Events, GatewayIntentBits, MessageReaction, PartialMessageReaction, Partials, User } from 'discord.js'; -import { db, openDb, reactionEmojis, recordReaction } from './util'; +import { db, openDb, reactionEmojis, recordReaction, sync } from './util'; const client = new Client({ intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildMessages, GatewayIntentBits.GuildMessageReactions], @@ -57,6 +57,7 @@ async function startup() { console.log("[db] Ready."); console.log("[bot] Logging in..."); await client.login(process.env.TOKEN); + await sync(client.guilds); } startup(); diff --git a/discord/sync.ts b/discord/sync.ts index 8117309..766400f 100644 --- a/discord/sync.ts +++ b/discord/sync.ts @@ -1,20 +1,11 @@ /** * sync.ts - * Syncs the message reactions in chat with the database, for when the bot is not running. + * Syncs message reactions in chat with the database. + * The bot will do the same thing automatically upon startup, but this is just in the form of a standalone script. */ -import { - Client, - Collection, - Events, - GatewayIntentBits, - GuildTextBasedChannel, - IntentsBitField, - Message, - MessageReaction, - Partials -} from 'discord.js'; -import { db, clearDb, openDb, reactionEmojis, recordReaction } from './util'; +import { Client, Events, GatewayIntentBits, IntentsBitField, Partials } from 'discord.js'; +import { db, openDb, reactionEmojis, sync } from './util'; const client = new Client({ intents: [GatewayIntentBits.MessageContent, IntentsBitField.Flags.Guilds, IntentsBitField.Flags.GuildMessages], @@ -28,8 +19,6 @@ client.once(Events.ClientReady, async () => { }); async function startup() { - console.log("[db] Clearing database..."); - clearDb(); console.log("[db] Opening..."); await openDb(); console.log("[db] Migrating..."); @@ -37,39 +26,7 @@ async function startup() { console.log("[db] Ready."); console.log("[bot] Logging in..."); await client.login(process.env.TOKEN); - - const guild = await client.guilds.fetch(process.env.GUILD); - if (!guild) { - console.error(`[bot] FATAL: guild ${guild.id} not found!`); - return 1; - } - console.log(`[bot] Entered guild ${guild.id}`); - const channels = await guild.channels.fetch(); - const textChannels = > channels.filter(c => c && 'messages' in c && c.isTextBased); - for (const [id, textChannel] of textChannels) { - console.log(`[bot] Found text channel ${id}`); - let before: string = undefined; - let messages = new Collection>(); - let newMessages: Collection>; - try { - do { - newMessages = await textChannel.messages.fetch({before, limit: 100}); - messages = messages.concat(newMessages); - console.log(`[bot] [${id}] Fetched ${messages.size} messages (+${newMessages.size})`); - if (messages.size > 0) - before = messages.last().id; - } while (newMessages.size > 0); - console.log(`[bot] [${id}] Fetched all messages.`); - const reactions = messages.flatMap(m => m.reactions.cache); - console.log(`[bot] Found ${reactions.size} reactions`); - for (const [_, reaction] of reactions) { - await recordReaction(reaction); - } - console.log(`[bot] [${id}] Finished recording reactions.`); - } catch (err) { - console.warn(`[bot] [${id}] Failed to fetch messages and reactions: ${err}`); - } - } + await sync(client.guilds); process.exit(0); } diff --git a/discord/util.ts b/discord/util.ts index 213816e..75ffae7 100644 --- a/discord/util.ts +++ b/discord/util.ts @@ -3,7 +3,7 @@ * Common helper functions */ -import { MessageReaction, User } from 'discord.js'; +import { Collection, GuildManager, GuildTextBasedChannel, Message, MessageReaction, User } from 'discord.js'; import { createWriteStream, existsSync, unlinkSync } from 'fs'; import { get as httpGet } from 'https'; import { Database, open } from 'sqlite'; @@ -41,7 +41,7 @@ async function downloadUserAvatar(user: User) console.log(`[bot] Downloading ${user.id}'s avatar...`); const file = createWriteStream(userAvatarPath(user)); return new Promise(resolve => { - httpGet(user.avatarURL(), res => { + httpGet(user.displayAvatarURL(), res => { res.pipe(file); file.on('finish', () => { file.close(); @@ -83,7 +83,7 @@ async function recordReaction(reaction: MessageReaction) } try { await db.run( - `INSERT INTO messages(id, guild, channel, author, content, reaction_${emojiIdx}_count) VALUES(?, ?, ?, ?, ?, 1) + `INSERT INTO messages(id, guild, channel, author, content, reaction_${emojiIdx}_count) VALUES(?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET reaction_${emojiIdx}_count = ? WHERE id = ?`, reaction.message.id, reaction.message.guildId, @@ -91,6 +91,7 @@ async function recordReaction(reaction: MessageReaction) reaction.message.author.id, reaction.message.content, reaction.count, + reaction.count, reaction.message.id ); await refreshUserReactionTotalCount(reaction.message.author, emojiIdx); @@ -101,4 +102,62 @@ async function recordReaction(reaction: MessageReaction) } } -export { db, clearDb, openDb, reactionEmojis, recordReaction }; +async function sync(guilds: GuildManager) { + const guild = await guilds.fetch(process.env.GUILD); + if (!guild) { + console.error(`[bot] FATAL: guild ${guild.id} not found!`); + return 1; + } + console.log(`[bot] Entered guild ${guild.id}`); + const channels = await guild.channels.fetch(); + const textChannels = > channels.filter(c => c && 'messages' in c && c.isTextBased); + for (const [id, textChannel] of textChannels) { + console.log(`[bot] Found text channel ${id}`); + const oldestMsg = await db.get( + 'SELECT * FROM messages WHERE guild = ? AND channel = ? ORDER BY id ASC LIMIT 1', + guild.id, + id + ); + const newestMsg = await db.get( + 'SELECT * FROM messages WHERE guild = ? AND channel = ? ORDER BY id DESC LIMIT 1', + guild.id, + id + ); + let before: string = oldestMsg && String(oldestMsg.id); + let after: string = newestMsg && String(newestMsg.id); + let messagesCount = 0; + let reactionsCount = 0; + let newMessagesBefore: Collection>; + let newMessagesAfter: Collection>; + try { + do { + newMessagesBefore = await textChannel.messages.fetch({ before, limit: 100 }); + messagesCount += newMessagesBefore.size; + + newMessagesAfter = await textChannel.messages.fetch({ after, limit: 100 }); + messagesCount += newMessagesAfter.size; + console.log(`[bot] [${id}] Fetched ${messagesCount} messages (+${newMessagesBefore.size} older, ${newMessagesAfter.size} newer)`); + + const reactions = newMessagesBefore.flatMap(m => m.reactions.cache) + .concat(newMessagesAfter.flatMap(m => m.reactions.cache)); + for (const [_, reaction] of reactions) { + await recordReaction(reaction); + } + reactionsCount += reactions.size; + console.log(`[bot] [${id}] Recorded ${reactionsCount} reactions (+${reactions.size}).`); + + if (newMessagesBefore.size > 0) { + before = newMessagesBefore.last().id; + } + if (newMessagesAfter.size > 0) { + after = newMessagesAfter.first().id; + } + } while (newMessagesBefore.size === 100 || newMessagesAfter.size === 100); + console.log(`[bot] [${id}] Done.`); + } catch (err) { + console.warn(`[bot] [${id}] Failed to fetch messages and reactions: ${err}`); + } + } +} + +export { db, clearDb, openDb, reactionEmojis, recordReaction, sync }; diff --git a/models.ts b/models.ts index cc1c581..d47eac8 100644 --- a/models.ts +++ b/models.ts @@ -5,9 +5,9 @@ interface ScoreboardMessageRow { id: number, - author: number, guild: number, channel: number, + author: string, content: string, reaction_1_count: number, reaction_2_count: number, @@ -15,7 +15,7 @@ interface ScoreboardMessageRow { } interface ScoreboardUserRow { - id: number, + id: string, username: string, reaction_1_total: number, reaction_2_total: number, diff --git a/server.ts b/server.ts index 53631ac..b89c0e9 100644 --- a/server.ts +++ b/server.ts @@ -19,17 +19,20 @@ async function openDb() { return open({ filename: 'discord/db.sqlite', driver: Database3 - }) + }); } app.get('/', async (req, res) => { - const users = await db.all<[ScoreboardUserRow]>('SELECT * FROM users'); - //const msgs = await db.all<[ScoreboardMessageRow]>('SELECT * FROM messages'); + const msg1 = await db.all<[ScoreboardMessageRow]>('SELECT * FROM messages ORDER BY reaction_1_count DESC LIMIT 5'); + const msg2 = await db.all<[ScoreboardMessageRow]>('SELECT * FROM messages ORDER BY reaction_2_count DESC LIMIT 5'); + const msg3 = await db.all<[ScoreboardMessageRow]>('SELECT * FROM messages ORDER BY reaction_3_count DESC LIMIT 5'); + const bestMsg = await db.all<[ScoreboardMessageRow]>('SELECT *, SUM(reaction_1_count)+SUM(reaction_2_count)+SUM(reaction_3_count) AS all_reacts FROM messages GROUP BY id ORDER BY all_reacts DESC LIMIT 5'); - const funniest = [...users].sort((a, b) => a.reaction_1_total - b.reaction_1_total); - const realest = [...users].sort((a, b) => a.reaction_2_total - b.reaction_2_total); - const cunniest = [...users].sort((a, b) => a.reaction_3_total - b.reaction_3_total); - res.render('index', { funniest, realest, cunniest }); + const funniest = await db.all<[ScoreboardUserRow]>('SELECT * FROM users ORDER BY reaction_1_total DESC'); + const realest = await db.all<[ScoreboardUserRow]>('SELECT * FROM users ORDER BY reaction_2_total DESC'); + const cunniest = await db.all<[ScoreboardUserRow]>('SELECT * FROM users ORDER BY reaction_3_total DESC'); + + res.render('index', { funniest, realest, cunniest, msg1, msg2, msg3, bestMsg }); }); app.listen(port, async () => { diff --git a/views/index.pug b/views/index.pug index ab99a05..26bb9ad 100644 --- a/views/index.pug +++ b/views/index.pug @@ -49,3 +49,84 @@ html img(src="/avatars/" + row.id + ".webp", height="64") td span= row.reaction_3_total + br + h1 Funniest Messages 💀 + table + tbody + each row in msg1 + tr + td + img(src="/avatars/" + row.author + ".webp", height="48") + td + if row.content + span= row.content + else + span + i (media) + tr + td + a(href="https://discord.com/channels/" + row.guild + "/" + row.channel + "/" + row.id) Link + td + span= "💀 " + row.reaction_1_count + br + h1 Realest Messages 💯 + table + tbody + each row in msg2 + tr + td + img(src="/avatars/" + row.author + ".webp", height="48") + td + if row.content + span= row.content + else + span + i (media) + tr + td + a(href="https://discord.com/channels/" + row.guild + "/" + row.channel + "/" + row.id) Link + td + span= "💯 " + row.reaction_2_count + br + h1 Cunniest Messages 😭 + table + tbody + each row in msg3 + tr + td + img(src="/avatars/" + row.author + ".webp", height="48") + td + if row.content + span= row.content + else + span + i (media) + tr + td + a(href="https://discord.com/channels/" + row.guild + "/" + row.channel + "/" + row.id) Link + td + span= "😭 " + row.reaction_3_count + br + h1 Best Messages 💀💯😭 + table + tbody + each row in bestMsg + tr + td + img(src="/avatars/" + row.author + ".webp", height="48") + td + if row.content + span= row.content + else + span + i (media) + tr + td + a(href="https://discord.com/channels/" + row.guild + "/" + row.channel + "/" + row.id) Link + td + if row.reaction_1_count + span= " 💀 " + row.reaction_1_count + if row.reaction_2_count + span= " 💯 " + row.reaction_2_count + if row.reaction_3_count + span= " 😭 " + row.reaction_3_count \ No newline at end of file