Fetch csv files from date range input and use setTimeout
This commit is contained in:
@@ -6,6 +6,7 @@ import fetch.{FetchService, FileNameService}
|
|||||||
|
|
||||||
object Main extends IOApp {
|
object Main extends IOApp {
|
||||||
def run(args: List[String]): IO[ExitCode] = {
|
def run(args: List[String]): IO[ExitCode] = {
|
||||||
|
// TODO looks like fetchTask fails server if no csv on meteo server
|
||||||
val fetchTask = FileNameService.generateCurrentHour.flatMap(FetchService.fetchSingleFile)
|
val fetchTask = FileNameService.generateCurrentHour.flatMap(FetchService.fetchSingleFile)
|
||||||
val scheduler = Scheduler.scheduleTask(fetchTask)
|
val scheduler = Scheduler.scheduleTask(fetchTask)
|
||||||
.evalMap { case (name, content) =>
|
.evalMap { case (name, content) =>
|
||||||
|
|||||||
@@ -51,7 +51,8 @@ object Server {
|
|||||||
(fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
|
(fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
|
||||||
saveResults <- successDownloads.traverse { case (name, content) => DBService.save(name, content) }
|
saveResults <- successDownloads.traverse { case (name, content) => DBService.save(name, content) }
|
||||||
(saveErrors, successSaves) = saveResults.partitionMap(identity)
|
(saveErrors, successSaves) = saveResults.partitionMap(identity)
|
||||||
successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
|
// successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
|
||||||
|
successes = successSaves
|
||||||
errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
|
errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
|
||||||
_ <- IO.println(s"errors: $errors")
|
_ <- IO.println(s"errors: $errors")
|
||||||
_ <- IO.println(s"successes: $successes")
|
_ <- IO.println(s"successes: $successes")
|
||||||
|
|||||||
+1
-1
@@ -10,7 +10,7 @@ console.log("api host:", import.meta.env.VITE_API_HOST);
|
|||||||
type Section = "aggregator" | "fileManager";
|
type Section = "aggregator" | "fileManager";
|
||||||
|
|
||||||
const App: Component = () => {
|
const App: Component = () => {
|
||||||
const [getSection, setSection] = createSignal<Section>("aggregator");
|
const [getSection, setSection] = createSignal<Section>("fileManager");
|
||||||
|
|
||||||
const section = () => getSection() === "aggregator" ? <Aggregator /> : <FileManager />;
|
const section = () => getSection() === "aggregator" ? <Aggregator /> : <FileManager />;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import moment from "moment";
|
||||||
|
import { createSignal } from "solid-js";
|
||||||
|
import { apiHost } from "../consts";
|
||||||
|
|
||||||
|
export function FetchFiles() {
|
||||||
|
const [getStart, setStart] = createSignal(new Date());
|
||||||
|
const [getEnd, setEnd] = createSignal(new Date());
|
||||||
|
const [getFetchResult, setFetchResult] = createSignal("");
|
||||||
|
const [isSpinner, setIsSpinner] = createSignal(false);
|
||||||
|
|
||||||
|
const startStr = () => moment(getStart()).format("YYYY-MM-DD");
|
||||||
|
const endStr = () => moment(getEnd()).format("YYYY-MM-DD");
|
||||||
|
|
||||||
|
function handleSubmit(e: MouseEvent) {
|
||||||
|
e.preventDefault();
|
||||||
|
const dates = getDatesBetween(getStart(), getEnd());
|
||||||
|
timerFetch(dates);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function timerFetch(dates: Date[]) {
|
||||||
|
setIsSpinner(true);
|
||||||
|
for (const date of dates) {
|
||||||
|
const response = await fetch(`${apiHost}/api/fetch/date/${moment(date).format("YYYYMMDD")}`);
|
||||||
|
const text = await response.text();
|
||||||
|
setFetchResult(text);
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 200));
|
||||||
|
}
|
||||||
|
setIsSpinner(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<form>
|
||||||
|
<h3>Date range fetch</h3>
|
||||||
|
<p>
|
||||||
|
<input
|
||||||
|
type="date"
|
||||||
|
value={ startStr() }
|
||||||
|
onChange={e => setStart(new Date(e.target.value))}
|
||||||
|
/> start
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
<input
|
||||||
|
type="date"
|
||||||
|
value={ endStr() }
|
||||||
|
onChange={e => setEnd(new Date(e.target.value))}
|
||||||
|
/> end
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
<input type="submit" onClick={handleSubmit} value="Fetch .csv files" />
|
||||||
|
</p>
|
||||||
|
<p>result:</p>
|
||||||
|
{ isSpinner() && <div>
|
||||||
|
<span class="spinner"></span>
|
||||||
|
<span style={{ "padding-left": "16px" }}>Fetching csv files</span>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
<p>{ getFetchResult() }</p>
|
||||||
|
</form>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDatesBetween(startDate: Date, endDate: Date): Date[] {
|
||||||
|
const dates: Date[] = [];
|
||||||
|
let currentDate = new Date(startDate); // start from the start date
|
||||||
|
|
||||||
|
while (currentDate <= endDate) {
|
||||||
|
dates.push(new Date(currentDate)); // add current date to the list
|
||||||
|
currentDate.setDate(currentDate.getDate() + 1); // increment the date
|
||||||
|
}
|
||||||
|
|
||||||
|
return dates;
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import { createSignal } from "solid-js";
|
import { createSignal } from "solid-js";
|
||||||
import "../css/FileManager.css"
|
import "../css/FileManager.css"
|
||||||
import { DateList } from "./DateList";
|
import { DateList } from "./DateList";
|
||||||
|
import { FetchFiles } from "./FetchFiles";
|
||||||
import { FileContent } from "./FileContent";
|
import { FileContent } from "./FileContent";
|
||||||
import { FileNameList } from "./FileNameList";
|
import { FileNameList } from "./FileNameList";
|
||||||
|
|
||||||
@@ -13,6 +14,7 @@ export function FileManager() {
|
|||||||
<h2>File manager</h2>
|
<h2>File manager</h2>
|
||||||
<div class="container">
|
<div class="container">
|
||||||
<div class="column">
|
<div class="column">
|
||||||
|
<FetchFiles />
|
||||||
<DateList setDate={setDate} />
|
<DateList setDate={setDate} />
|
||||||
</div>
|
</div>
|
||||||
<div class="column">
|
<div class="column">
|
||||||
|
|||||||
Reference in New Issue
Block a user