Skip to content

Commit 39a9a26

Browse files
bluescorpianbluescorpian
committed
allow download specefic years/months and stream the csv file
1 parent 56ebc6a commit 39a9a26

File tree

4 files changed

+198
-42
lines changed

4 files changed

+198
-42
lines changed

‎backend/main.ts

+113-33
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { StationStats, type TableDataItem } from "../types.d.ts";
22
import { getWSData } from "./wind2speed.ts";
3-
import { Application, Router } from "@oak/oak";
3+
import { Application, Context, Router } from "@oak/oak";
44
import { oakCors } from "https://deno.land/x/[email protected]/mod.ts";
55
import { stringify } from "jsr:@std/csv/stringify";
66

@@ -41,6 +41,7 @@ async function downloadWindData(stationId: number) {
4141
"station",
4242
data.station.id,
4343
]);
44+
const months = stationStats.value?.months || [];
4445
const latestEntryTimestamp = stationStats.value?.latestEntryTimestamp || 0;
4546

4647
const newTableData = data.tableData.filter(
@@ -53,18 +54,28 @@ async function downloadWindData(stationId: number) {
5354
const transaction = kv.atomic();
5455

5556
for (const entry of newTableData) {
57+
const date = parseObsTimeLocal(entry.obsTimeLocal);
58+
const year = date.getFullYear();
59+
const month = date.getMonth();
60+
61+
if (!months.some((m) => m.year === year && m.month === month)) {
62+
months.push({ year, month });
63+
}
64+
5665
transaction.set(
57-
["windHistoryData", data.station.id, entry.id],
66+
["windHistoryData", data.station.id, year, month, entry.id],
5867
entry
5968
);
6069
}
6170
transaction.set(["station", data.station.id], {
6271
...data.station,
63-
entries: (stationStats.value?.entries ?? 0) + newTableData.length,
72+
totalEntries:
73+
(stationStats.value?.totalEntries ?? 0) + newTableData.length,
6474
latestEntryTimestamp: parseObsTimeLocal(
6575
newTableData[0].obsTimeLocal
6676
).getTime(),
67-
});
77+
months,
78+
} as StationStats);
6879
transaction.set(["latestUpdatedStation"], stationId);
6980

7081
const result = await transaction.commit();
@@ -86,6 +97,20 @@ function parseObsTimeLocal(obsTimeLocal: string): Date {
8697

8798
const router = new Router();
8899

100+
const authMiddleware = async (ctx: Context, next: () => Promise<unknown>) => {
101+
if (
102+
(ctx.request.headers.get("Authorization") ?? "") !==
103+
(Deno.env.get("PASSWORD") ?? "")
104+
) {
105+
ctx.response.status = 401;
106+
ctx.response.body = {
107+
msg: "Unauthorized",
108+
};
109+
return;
110+
}
111+
await next();
112+
};
113+
89114
router.post("/login", async (ctx) => {
90115
const body = await ctx.request.body.text();
91116
const formData = new URLSearchParams(body);
@@ -104,18 +129,7 @@ router.get("/tracked-stations", async (ctx) => {
104129
ctx.response.body = trackedStations ?? [];
105130
});
106131

107-
router.post("/tracked-stations", async (ctx) => {
108-
if (
109-
(ctx.request.headers.get("Authorization") ?? "") !==
110-
(Deno.env.get("PASSWORD") ?? "")
111-
) {
112-
ctx.response.status = 401;
113-
ctx.response.body = {
114-
msg: "Unauthorized",
115-
};
116-
return;
117-
}
118-
132+
router.post("/tracked-stations", authMiddleware, async (ctx) => {
119133
try {
120134
const body = await ctx.request.body.text();
121135
const formData = new URLSearchParams(body);
@@ -140,6 +154,35 @@ router.post("/tracked-stations", async (ctx) => {
140154
}
141155
});
142156

157+
router.post(
158+
"/update-station-months/:stationId",
159+
authMiddleware,
160+
async (ctx) => {
161+
const stationId = Number(ctx.params.stationId);
162+
const stationStats = await kv.get<StationStats>(["station", stationId]);
163+
const tableDataEntries = await kv.list<TableDataItem>({
164+
prefix: ["windHistoryData", stationId],
165+
});
166+
167+
for await (const entry of tableDataEntries) {
168+
const date = parseObsTimeLocal(entry.value.obsTimeLocal);
169+
const year = date.getFullYear();
170+
const month = date.getMonth();
171+
172+
if (
173+
!stationStats.value?.months.some(
174+
(m) => m.year === year && m.month === month
175+
)
176+
) {
177+
stationStats.value?.months.push({ year, month });
178+
}
179+
}
180+
await kv.set(["station", stationId], stationStats.value);
181+
182+
ctx.response.status = 200;
183+
}
184+
);
185+
143186
router.get("/stations", async (ctx) => {
144187
const stationsEntries = await kv.list<StationStats>({
145188
prefix: ["station"],
@@ -152,20 +195,35 @@ router.get("/stations", async (ctx) => {
152195
ctx.response.body = stations;
153196
});
154197

155-
router.get("/wind-history/:stationId/csv", async (ctx) => {
198+
router.get("/wind-history/:stationId", async (ctx) => {
156199
const stationId = Number(ctx.params.stationId);
200+
const fileformat = ctx.request.url.searchParams.get("fileformat") || "csv";
201+
const year = Number(ctx.request.url.searchParams.get("year"));
202+
const month = Number(ctx.request.url.searchParams.get("month"));
203+
let datePrefix = [];
204+
if (year) {
205+
datePrefix.push(year);
206+
if (month) datePrefix.push(month);
207+
}
208+
209+
const filenameParts = ["wind-history", stationId.toString()];
210+
if (year) {
211+
filenameParts.push(year.toString());
212+
if (month) {
213+
filenameParts.push(month.toString());
214+
}
215+
}
216+
const filename = filenameParts.join("-") + ".csv";
217+
157218
const entries = await kv.list<TableDataItem>({
158-
prefix: ["windHistoryData", stationId],
219+
prefix: ["windHistoryData", stationId, ...datePrefix],
159220
});
160-
const data: TableDataItem[] = [];
161-
for await (const entry of entries) {
162-
data.push(entry.value);
163-
}
164-
const csv = await stringify(
165-
data as unknown as readonly Record<string, unknown>[],
166-
{
167-
headers: true,
168-
columns: [
221+
222+
const encoder = new TextEncoder();
223+
const stream = new ReadableStream({
224+
async start(controller) {
225+
// Write CSV headers
226+
const headers = [
169227
"id",
170228
"stationId",
171229
"obsTimeLocal",
@@ -178,16 +236,38 @@ router.get("/wind-history/:stationId/csv", async (ctx) => {
178236
"humidityAvg",
179237
"tempAvg",
180238
"pressureAvg",
181-
],
182-
}
183-
);
239+
];
240+
controller.enqueue(encoder.encode(headers.join(",") + "\n"));
241+
242+
// Write CSV data rows
243+
for await (const entry of entries) {
244+
const row = [
245+
entry.value.id,
246+
entry.value.stationId,
247+
entry.value.obsTimeLocal,
248+
entry.value.winddirHigh,
249+
entry.value.winddirLow,
250+
entry.value.winddirAvg,
251+
entry.value.windspeedHigh,
252+
entry.value.windspeedAvg,
253+
entry.value.windspeedLow,
254+
entry.value.humidityAvg,
255+
entry.value.tempAvg,
256+
entry.value.pressureAvg,
257+
];
258+
controller.enqueue(encoder.encode(row.join(",") + "\n"));
259+
}
260+
261+
controller.close();
262+
},
263+
});
264+
184265
ctx.response.headers.set("Content-Type", "text/csv");
185-
const date = new Date().toISOString().split("T")[0];
186266
ctx.response.headers.set(
187267
"Content-Disposition",
188-
`attachment; filename="wind-history-${stationId}-${date}.csv"`
268+
`attachment; filename="${filename}"`
189269
);
190-
ctx.response.body = csv;
270+
ctx.response.body = stream;
191271
});
192272

193273
const app = new Application();

‎frontend/src/lib/DownloadModal.svelte

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
<script lang="ts">
2+
import {
3+
ListGroup,
4+
ListGroupItem,
5+
Modal,
6+
ModalBody,
7+
ModalHeader,
8+
Button,
9+
} from "@sveltestrap/sveltestrap";
10+
import type { StationStats } from "../../../types";
11+
import moment from "moment";
12+
interface Props {
13+
station: StationStats;
14+
isOpen: boolean;
15+
}
16+
17+
let { station, isOpen = $bindable() }: Props = $props();
18+
19+
const years = new Set(station.months.map((m) => m.year));
20+
</script>
21+
22+
<Modal {isOpen} centered={true} toggle={() => (isOpen = !isOpen)}>
23+
<ModalHeader>Download wind history</ModalHeader>
24+
<ModalBody>
25+
<p>{station.nam} ({station.id})</p>
26+
<ListGroup>
27+
{#each years as year}
28+
<ListGroupItem
29+
class="d-flex justify-content-between align-items-center"
30+
color={"primary"}
31+
>
32+
<strong>{year}</strong>
33+
<Button
34+
size="sm"
35+
color="primary"
36+
href={import.meta.env.VITE_API_URL +
37+
`/wind-history/${station.id}?fileformat=csv&year=${year}`}
38+
>CSV</Button
39+
>
40+
</ListGroupItem>
41+
{#each station.months
42+
.filter((m) => m.year === year)
43+
.map((m) => m.month)
44+
.sort((a, b) => a - b) as month}
45+
<ListGroupItem
46+
class="d-flex justify-content-between align-items-center"
47+
>
48+
<span>
49+
{year} / {moment().month(month).format("MMM")}
50+
</span>
51+
52+
<Button
53+
size="sm"
54+
outline
55+
color="secondary"
56+
href={import.meta.env.VITE_API_URL +
57+
`/wind-history/${station.id}?fileformat=csv&year=${year}&month=${month}`}
58+
>CSV</Button
59+
>
60+
</ListGroupItem>
61+
{/each}
62+
{/each}
63+
</ListGroup>
64+
</ModalBody>
65+
</Modal>

‎frontend/src/lib/Stations.svelte

+18-8
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@
99
ListGroupItem,
1010
} from "@sveltestrap/sveltestrap";
1111
import { type StationStats } from "../../../types";
12-
1312
import moment from "moment";
13+
import DownloadModal from "./DownloadModal.svelte";
1414
1515
let stations: StationStats[] = $state([]);
16+
let downloadOpen: boolean = $state(false);
17+
let downloadStation: StationStats | null = $state(null);
1618
$effect(() => {
1719
fetch(import.meta.env.VITE_API_URL + "/stations")
1820
.then(async (res) => {
@@ -38,15 +40,23 @@
3840
).fromNow()}</ListGroupItem
3941
>
4042
<ListGroupItem
41-
><strong>Total Entries</strong>: {station.entries}</ListGroupItem
43+
><strong>Total Entries</strong>: {station.totalEntries}</ListGroupItem
4244
>
4345
</ListGroup>
44-
<CardFooter
45-
><Button
46-
href={import.meta.env.VITE_API_URL +
47-
`/wind-history/${station.id}/csv`}>CSV</Button
48-
></CardFooter
49-
>
46+
<CardFooter>
47+
<Button
48+
color="secondary"
49+
outline
50+
onclick={() => {
51+
downloadStation = station;
52+
downloadOpen = true;
53+
}}>Download</Button
54+
>
55+
</CardFooter>
5056
</Card>
5157
</Col>
5258
{/each}
59+
{#if downloadStation}
60+
<DownloadModal bind:isOpen={downloadOpen} station={downloadStation}
61+
></DownloadModal>
62+
{/if}

‎types.d.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ export interface WindData {
8181
}
8282

8383
export interface StationStats extends Station {
84-
entries: number;
84+
totalEntries: number;
85+
months: { year: number; month: number }[];
8586
latestEntryTimestamp: number;
8687
}

0 commit comments

Comments
 (0)