Skip to content

Commit b083d21

Browse files
cp-20cp-20
committed
🦪 🧂 ☕ script 周りのリファクタ + update all を追加
1 parent 25735b5 commit b083d21

11 files changed

+255
-225
lines changed

‎apps/server/package.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@
1313
"lint": "eslint ./src/**/*.ts ./src/**/*.tsx",
1414
"build": "tsup --config tsup.config.ts",
1515
"start": "bun run ./src/index.ts",
16-
"update-messages": "bun run ./src/traQ/update-messages.ts",
17-
"update-statistics": "bun run ./src/traQ/update-statistics.ts"
16+
"update-messages": "bun run ./src/traQ/scripts/update-messages.ts",
17+
"update-all-messages": "bun run ./src/traQ/scripts/update-all-messages.ts",
18+
"update-statistics": "bun run ./src/traQ/scripts/update-statistics.ts"
1819
},
1920
"dependencies": {
2021
"@hono/zod-validator": "^0.2.2",

‎apps/server/src/traQ/drop-messages-and-stamps.ts

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import { dropMessages, dropStamps } from '@traq-ing/database';
2+
3+
export const dropMessagesAndStamps = async () => {
4+
await dropMessages();
5+
await dropStamps();
6+
};

‎apps/server/src/traQ/index.ts

+3-219
Original file line numberDiff line numberDiff line change
@@ -1,219 +1,3 @@
1-
import {
2-
deleteChannelPins,
3-
deleteChannelSubscriptions,
4-
deleteTags,
5-
deleteUserGroupRelations,
6-
getChannelPins,
7-
getChannelSubscriptions,
8-
getLastMessageCreatedAt,
9-
getTags,
10-
getUserGroupRelations,
11-
insertChannelPins,
12-
insertChannelSubscriptions,
13-
insertMessageStamps,
14-
insertMessages,
15-
insertTags,
16-
insertUserGroupRelations,
17-
updateMaterializedViews,
18-
} from '@traq-ing/database';
19-
import { api } from './api';
20-
21-
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
22-
23-
export const updateMessages = async () => {
24-
const lastMessage = await getLastMessageCreatedAt();
25-
const after = lastMessage ? new Date(lastMessage.getTime() - 1000 * 60 * 60 * 24 * 7).toISOString() : undefined;
26-
let before = new Date().toISOString();
27-
28-
const res = await api.users.getUsers();
29-
if (!res.ok) {
30-
throw new Error('Failed to fetch users');
31-
}
32-
const users = res.data;
33-
const isBot = (id: string) => users.find((u) => u.id === id)?.bot ?? false;
34-
35-
let offset = 0;
36-
while (true) {
37-
const res = await api.messages.searchMessages({
38-
sort: 'createdAt',
39-
limit: 100,
40-
after,
41-
before,
42-
});
43-
if (!res.ok) continue;
44-
45-
const searched = res.data.hits;
46-
if (searched.length === 0) break;
47-
48-
const messages = searched.map((m) => ({
49-
id: m.id,
50-
userId: m.userId,
51-
channelId: m.channelId,
52-
content: m.content.replaceAll('\u0000', ''),
53-
createdAt: new Date(m.createdAt),
54-
pinned: m.pinned,
55-
isBot: isBot(m.userId),
56-
}));
57-
58-
await insertMessages(messages);
59-
60-
const stamps = searched.flatMap((m) =>
61-
m.stamps.map((s) => ({
62-
messageId: m.id,
63-
userId: s.userId,
64-
stampId: s.stampId,
65-
channelId: m.channelId,
66-
messageUserId: m.userId,
67-
count: s.count,
68-
createdAt: new Date(s.createdAt),
69-
isBot: isBot(m.userId),
70-
isBotMessage: isBot(s.userId),
71-
})),
72-
);
73-
74-
if (stamps.length > 0) {
75-
await insertMessageStamps(stamps);
76-
}
77-
78-
offset += searched.length;
79-
before = searched[searched.length - 1].createdAt;
80-
81-
console.log(offset, searched[searched.length - 1].createdAt);
82-
await sleep(100);
83-
}
84-
85-
await updateMaterializedViews();
86-
};
87-
88-
const getDiff = <T>(before: T[], after: T[], comp: (a: T, b: T) => boolean) => {
89-
const added = after.filter((a) => !before.some((b) => comp(a, b)));
90-
const deleted = before.filter((b) => !after.some((a) => comp(a, b)));
91-
return { added, deleted };
92-
};
93-
94-
const chunkArray = <T>(array: T[], size = 1000) => {
95-
const chunks = [];
96-
for (let i = 0; i < array.length; i += size) {
97-
chunks.push(array.slice(i, i + size));
98-
}
99-
return chunks;
100-
};
101-
102-
export const updateStatistics = async () => {
103-
// users
104-
const userRes = await api.users.getUsers({ 'include-suspended': true });
105-
if (!userRes.ok) throw new Error('Failed to fetch users');
106-
const users = userRes.data;
107-
console.log(`Successfully fetched and inserted ${users.length} users`);
108-
109-
// groups
110-
const groupRes = await api.groups.getUserGroups();
111-
if (!groupRes.ok) throw new Error('Failed to fetch groups');
112-
const groups = groupRes.data;
113-
const insertedUserGroupRelations = await getUserGroupRelations();
114-
console.log(`Successfully fetched ${groups.length} groups`);
115-
116-
const newUserGroupRelations = groups.flatMap((group) =>
117-
group.members.map((user) => ({
118-
userId: user.id,
119-
groupId: group.id,
120-
isAdmin: group.admins.includes(user.id),
121-
})),
122-
);
123-
const userGroupRelationsDiff = getDiff(
124-
insertedUserGroupRelations,
125-
newUserGroupRelations,
126-
(a, b) => a.userId === b.userId && a.groupId === b.groupId,
127-
);
128-
if (userGroupRelationsDiff.added.length > 0) await insertUserGroupRelations(userGroupRelationsDiff.added);
129-
console.log(`Successfully inserted ${userGroupRelationsDiff.added.length} groups`);
130-
if (userGroupRelationsDiff.deleted.length > 0) await deleteUserGroupRelations(userGroupRelationsDiff.deleted);
131-
console.log(`Successfully deleted ${userGroupRelationsDiff.deleted.length} groups`);
132-
133-
// tags
134-
const newTags = [];
135-
for (const user of users) {
136-
const res = await api.users.getUser(user.id);
137-
if (!res.ok) throw new Error('Failed to fetch user');
138-
const userData = res.data;
139-
newTags.push(
140-
...userData.tags.map((tag) => ({
141-
userId: user.id,
142-
name: tag.tag,
143-
})),
144-
);
145-
await sleep(100);
146-
}
147-
console.log(`Successfully fetched ${newTags.length} tags`);
148-
const insertedTags = await getTags();
149-
const tagsDiff = getDiff(insertedTags, newTags, (a, b) => a.userId === b.userId && a.name === b.name);
150-
if (tagsDiff.added.length > 0) await insertTags(tagsDiff.added);
151-
console.log(`Successfully inserted ${tagsDiff.added.length} tags`);
152-
if (tagsDiff.deleted.length > 0) await deleteTags(tagsDiff.deleted);
153-
console.log(`Successfully deleted ${tagsDiff.deleted.length} tags`);
154-
155-
// channels
156-
const channelRes = await api.channels.getChannels();
157-
if (!channelRes.ok) throw new Error('Failed to fetch channels');
158-
const channels = channelRes.data.public;
159-
console.log(`Successfully fetched ${channels.length} channels`);
160-
161-
// channel subscriptions
162-
const newChannelSubscriptions = [];
163-
for (const channel of channels) {
164-
try {
165-
const res = await api.channels.getChannelSubscribers(channel.id);
166-
if (!res.ok) throw new Error('Failed to fetch channel subscriptions');
167-
const subscriptions = res.data;
168-
newChannelSubscriptions.push(
169-
...subscriptions.map((subscription) => ({
170-
userId: subscription,
171-
channelId: channel.id,
172-
})),
173-
);
174-
} catch (err) {
175-
// 403エラーの場合はスキップ
176-
}
177-
await sleep(100);
178-
}
179-
const insertedChannelSubscriptions = await getChannelSubscriptions();
180-
const channelSubscriptionsDiff = getDiff(
181-
insertedChannelSubscriptions,
182-
newChannelSubscriptions,
183-
(a, b) => a.userId === b.userId && a.channelId === b.channelId,
184-
);
185-
// 一度に大量のデータを挿入するとエラーが発生するため、1000件ずつ挿入する
186-
for (const chunk of chunkArray(channelSubscriptionsDiff.added)) {
187-
await insertChannelSubscriptions(chunk);
188-
}
189-
console.log(`Successfully inserted ${channelSubscriptionsDiff.added.length} channel subscriptions`);
190-
if (channelSubscriptionsDiff.deleted.length > 0) await deleteChannelSubscriptions(channelSubscriptionsDiff.deleted);
191-
console.log(`Successfully deleted ${channelSubscriptionsDiff.deleted.length} channel subscriptions`);
192-
193-
// channel pins
194-
const newChannelPins = [];
195-
for (const channel of channels) {
196-
const res = await api.channels.getChannelPins(channel.id);
197-
if (!res.ok) throw new Error('Failed to fetch channel pins');
198-
const pins = res.data;
199-
newChannelPins.push(
200-
...pins.map((pin) => ({
201-
channelId: channel.id,
202-
messageId: pin.message.id,
203-
})),
204-
);
205-
await sleep(100);
206-
}
207-
const insertedChannelPins = await getChannelPins();
208-
const channelPinsDiff = getDiff(
209-
insertedChannelPins,
210-
newChannelPins,
211-
(a, b) => a.channelId === b.channelId && a.messageId === b.messageId,
212-
);
213-
for (const chunk of chunkArray(channelPinsDiff.added)) {
214-
await insertChannelPins(chunk);
215-
}
216-
console.log(`Successfully inserted ${channelPinsDiff.added.length} channel pins`);
217-
if (channelPinsDiff.deleted.length > 0) await deleteChannelPins(channelPinsDiff.deleted);
218-
console.log(`Successfully deleted ${channelPinsDiff.deleted.length} channel pins`);
219-
};
1+
export * from './update-messages';
2+
export * from './update-statistics';
3+
export * from './drop-messages-and-stamps';

‎apps/server/src/traQ/scripts/update-all-messages.ts

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
import { dropMessagesAndStamps, updateMessages } from '@/traQ';
2+
3+
await dropMessagesAndStamps();
4+
await updateMessages(true);

‎apps/server/src/traQ/scripts/update-messages.ts

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import { updateMessages } from '..';
2+
3+
await updateMessages(false);

‎apps/server/src/traQ/scripts/update-statistics.ts

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import { updateStatistics } from '..';
2+
3+
await updateStatistics();

‎apps/server/src/traQ/update-messages.ts

+76-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,77 @@
1-
import { updateMessages } from '.';
1+
import { api } from '@/traQ/api';
2+
import {
3+
getLastMessageCreatedAt,
4+
insertMessages,
5+
insertMessageStamps,
6+
updateMaterializedViews,
7+
} from '@traq-ing/database';
8+
import { sleep } from 'bun';
29

3-
await updateMessages();
10+
export const updateMessages = async (getAll: boolean) => {
11+
const after = await (async () => {
12+
if (getAll) return undefined;
13+
const lastMessage = await getLastMessageCreatedAt();
14+
const after = lastMessage ? new Date(lastMessage.getTime() - 1000 * 60 * 60 * 24 * 7).toISOString() : undefined;
15+
return after;
16+
})();
17+
let before = new Date().toISOString();
18+
19+
const res = await api.users.getUsers();
20+
if (!res.ok) {
21+
throw new Error('Failed to fetch users');
22+
}
23+
const users = res.data;
24+
const isBot = (id: string) => users.find((u) => u.id === id)?.bot ?? false;
25+
26+
let offset = 0;
27+
while (true) {
28+
const res = await api.messages.searchMessages({
29+
sort: 'createdAt',
30+
limit: 100,
31+
after,
32+
before,
33+
});
34+
if (!res.ok) continue;
35+
36+
const searched = res.data.hits;
37+
if (searched.length === 0) break;
38+
39+
const messages = searched.map((m) => ({
40+
id: m.id,
41+
userId: m.userId,
42+
channelId: m.channelId,
43+
content: m.content.replaceAll('\u0000', ''),
44+
createdAt: new Date(m.createdAt),
45+
pinned: m.pinned,
46+
isBot: isBot(m.userId),
47+
}));
48+
49+
await insertMessages(messages);
50+
51+
const stamps = searched.flatMap((m) =>
52+
m.stamps.map((s) => ({
53+
messageId: m.id,
54+
userId: s.userId,
55+
stampId: s.stampId,
56+
channelId: m.channelId,
57+
messageUserId: m.userId,
58+
count: s.count,
59+
createdAt: new Date(s.createdAt),
60+
isBot: isBot(m.userId),
61+
isBotMessage: isBot(s.userId),
62+
})),
63+
);
64+
65+
if (stamps.length > 0) {
66+
await insertMessageStamps(stamps);
67+
}
68+
69+
offset += searched.length;
70+
before = searched[searched.length - 1].createdAt;
71+
72+
console.log(offset, searched[searched.length - 1].createdAt);
73+
await sleep(100);
74+
}
75+
76+
await updateMaterializedViews();
77+
};

0 commit comments

Comments
 (0)