Fetch csv files from date range input and use setTimeout

This commit is contained in:
Guntis Smaukstelis
2023-05-17 16:28:56 +03:00
parent affc2e0d93
commit ff3efdfb58
5 changed files with 78 additions and 2 deletions
+1
View File
@@ -6,6 +6,7 @@ import fetch.{FetchService, FileNameService}
object Main extends IOApp {
def run(args: List[String]): IO[ExitCode] = {
// TODO looks like fetchTask fails server if no csv on meteo server
val fetchTask = FileNameService.generateCurrentHour.flatMap(FetchService.fetchSingleFile)
val scheduler = Scheduler.scheduleTask(fetchTask)
.evalMap { case (name, content) =>
+2 -1
View File
@@ -51,7 +51,8 @@ object Server {
(fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
saveResults <- successDownloads.traverse { case (name, content) => DBService.save(name, content) }
(saveErrors, successSaves) = saveResults.partitionMap(identity)
successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
// successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
successes = successSaves
errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
_ <- IO.println(s"errors: $errors")
_ <- IO.println(s"successes: $successes")
+1 -1
View File
@@ -10,7 +10,7 @@ console.log("api host:", import.meta.env.VITE_API_HOST);
type Section = "aggregator" | "fileManager";
const App: Component = () => {
const [getSection, setSection] = createSignal<Section>("aggregator");
const [getSection, setSection] = createSignal<Section>("fileManager");
const section = () => getSection() === "aggregator" ? <Aggregator /> : <FileManager />;
+72
View File
@@ -0,0 +1,72 @@
import moment from "moment";
import { createSignal } from "solid-js";
import { apiHost } from "../consts";
export function FetchFiles() {
const [getStart, setStart] = createSignal(new Date());
const [getEnd, setEnd] = createSignal(new Date());
const [getFetchResult, setFetchResult] = createSignal("");
const [isSpinner, setIsSpinner] = createSignal(false);
const startStr = () => moment(getStart()).format("YYYY-MM-DD");
const endStr = () => moment(getEnd()).format("YYYY-MM-DD");
function handleSubmit(e: MouseEvent) {
e.preventDefault();
const dates = getDatesBetween(getStart(), getEnd());
timerFetch(dates);
}
async function timerFetch(dates: Date[]) {
setIsSpinner(true);
for (const date of dates) {
const response = await fetch(`${apiHost}/api/fetch/date/${moment(date).format("YYYYMMDD")}`);
const text = await response.text();
setFetchResult(text);
await new Promise(resolve => setTimeout(resolve, 200));
}
setIsSpinner(false);
}
return (
<form>
<h3>Date range fetch</h3>
<p>
<input
type="date"
value={ startStr() }
onChange={e => setStart(new Date(e.target.value))}
/> start
</p>
<p>
<input
type="date"
value={ endStr() }
onChange={e => setEnd(new Date(e.target.value))}
/> end
</p>
<p>
<input type="submit" onClick={handleSubmit} value="Fetch .csv files" />
</p>
<p>result:</p>
{ isSpinner() && <div>
<span class="spinner"></span>
<span style={{ "padding-left": "16px" }}>Fetching csv files</span>
</div>
}
<p>{ getFetchResult() }</p>
</form>
)
}
function getDatesBetween(startDate: Date, endDate: Date): Date[] {
const dates: Date[] = [];
let currentDate = new Date(startDate); // start from the start date
while (currentDate <= endDate) {
dates.push(new Date(currentDate)); // add current date to the list
currentDate.setDate(currentDate.getDate() + 1); // increment the date
}
return dates;
}
+2
View File
@@ -1,6 +1,7 @@
import { createSignal } from "solid-js";
import "../css/FileManager.css"
import { DateList } from "./DateList";
import { FetchFiles } from "./FetchFiles";
import { FileContent } from "./FileContent";
import { FileNameList } from "./FileNameList";
@@ -13,6 +14,7 @@ export function FileManager() {
<h2>File manager</h2>
<div class="container">
<div class="column">
<FetchFiles />
<DateList setDate={setDate} />
</div>
<div class="column">