Skip to content
This repository was archived by the owner on Jun 2, 2025. It is now read-only.

Commit d7acea3

Browse files
peterdudfieldpeterdudfield
authored
Merge pull request #339 from openclimatefix/issue/visulaize
add visulization script
2 parents 2fc5f67 + 1b336d4 commit d7acea3

File tree

4 files changed

+300
-1
lines changed

4 files changed

+300
-1
lines changed

‎ocf_datapipes/visualization/README.md renamed to ‎ocf_datapipes/visualisation/README.md

File renamed without changes.

‎ocf_datapipes/visualization/__init__.py renamed to ‎ocf_datapipes/visualisation/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
"""Tools for plotting and visualizing data."""
2+
3+
from .batch import visualise_batch

‎ocf_datapipes/visualisation/batch.py

Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
""" The idea is visualise one of the batches
2+
3+
This is a bit of a work in progress, but the idea is to visualise the batch in a markdown file.
4+
"""
5+
6+
import os
7+
8+
import pandas as pd
9+
import plotly.graph_objects as go
10+
import torch
11+
12+
from ocf_datapipes.batch import BatchKey, NumpyBatch, NWPBatchKey
13+
14+
15+
def visualise_batch(batch: NumpyBatch, folder=".", output_file="report.md", limit_examples=None):
16+
"""Visualize the batch in a markdown file"""
17+
18+
# create dir if it does not exist
19+
for d in [folder, f"{folder}/gsp", f"{folder}/nwp", f"{folder}/satellite"]:
20+
if not os.path.exists(d):
21+
os.makedirs(d)
22+
23+
with open(f"{folder}/{output_file}", "a") as f:
24+
# Wind
25+
print("# Batch visualisation", file=f)
26+
27+
print("## Wind \n", file=f)
28+
keys = [
29+
BatchKey.wind,
30+
BatchKey.wind_t0_idx,
31+
BatchKey.wind_time_utc,
32+
BatchKey.wind_id,
33+
BatchKey.wind_observed_capacity_mwp,
34+
BatchKey.wind_nominal_capacity_mwp,
35+
BatchKey.wind_time_utc,
36+
BatchKey.wind_latitude,
37+
BatchKey.wind_longitude,
38+
BatchKey.wind_solar_azimuth,
39+
BatchKey.wind_solar_elevation,
40+
]
41+
for key in keys:
42+
if key in batch.keys():
43+
print("\n", file=f)
44+
value = batch[key]
45+
if isinstance(value, torch.Tensor):
46+
print(f"{key} {value.shape=}", file=f)
47+
print(f"Max {value.max()}", file=f)
48+
print(f"Min {value.min()}", file=f)
49+
elif isinstance(value, int):
50+
print(f"{key} {value}", file=f)
51+
else:
52+
print(f"{key} {value}", file=f)
53+
54+
print("## GSP \n", file=f)
55+
keys = [
56+
BatchKey.gsp,
57+
BatchKey.gsp_id,
58+
BatchKey.gsp_time_utc,
59+
BatchKey.gsp_time_utc_fourier,
60+
BatchKey.gsp_x_osgb,
61+
BatchKey.gsp_x_osgb_fourier,
62+
BatchKey.gsp_y_osgb,
63+
BatchKey.gsp_y_osgb_fourier,
64+
BatchKey.gsp_t0_idx,
65+
BatchKey.gsp_effective_capacity_mwp,
66+
BatchKey.gsp_nominal_capacity_mwp,
67+
BatchKey.gsp_solar_azimuth,
68+
BatchKey.gsp_solar_elevation,
69+
]
70+
for key in keys:
71+
if key in batch.keys():
72+
print("\n", file=f)
73+
print(f"### {key.name}", file=f)
74+
value = batch[key]
75+
if key.name == "gsp":
76+
# plot gsp data
77+
n_examples = value.shape[0]
78+
if limit_examples is not None:
79+
n_examples = min(n_examples, limit_examples)
80+
81+
for b in range(n_examples):
82+
fig = go.Figure()
83+
gsp_data = value[b, :, 0]
84+
time = pd.to_datetime(batch[BatchKey.gsp_time_utc][b], unit="s")
85+
fig.add_trace(go.Scatter(x=time, y=gsp_data, mode="lines", name="GSP"))
86+
fig.update_layout(
87+
title=f"GSP - example {b}", xaxis_title="Time", yaxis_title="Value"
88+
)
89+
# fig.show(renderer='browser')
90+
name = f"gsp/gsp_{b}.png"
91+
fig.write_image(f"{folder}/{name}")
92+
print(f"![](./{name})", file=f)
93+
print("\n", file=f)
94+
95+
elif isinstance(value, torch.Tensor):
96+
print(f"shape {value.shape=}", file=f)
97+
print(f"Max {value.max():.2f}", file=f)
98+
print(f"Min {value.min():.2f}", file=f)
99+
elif isinstance(value, int):
100+
print(f"{value}", file=f)
101+
else:
102+
print(f"{value}", file=f)
103+
104+
# TODO plot solar azimuth and elevation
105+
106+
# NWP
107+
print("## NWP \n", file=f)
108+
109+
keys = [
110+
NWPBatchKey.nwp,
111+
NWPBatchKey.nwp_target_time_utc,
112+
NWPBatchKey.nwp_channel_names,
113+
NWPBatchKey.nwp_step,
114+
NWPBatchKey.nwp_t0_idx,
115+
NWPBatchKey.nwp_init_time_utc,
116+
]
117+
118+
nwp = batch[BatchKey.nwp]
119+
120+
nwp_providers = nwp.keys()
121+
for provider in nwp_providers:
122+
print("\n", file=f)
123+
print(f"### Provider {provider}", file=f)
124+
nwp_provider = nwp[provider]
125+
126+
# plot nwp main data
127+
nwp_data = nwp_provider[NWPBatchKey.nwp]
128+
# average of lat and lon
129+
nwp_data = nwp_data.mean(dim=(3, 4))
130+
131+
n_examples = nwp_data.shape[0]
132+
if limit_examples is not None:
133+
n_examples = min(n_examples, limit_examples)
134+
135+
for b in range(n_examples):
136+
137+
fig = go.Figure()
138+
for i in range(len(nwp_provider[NWPBatchKey.nwp_channel_names])):
139+
channel = nwp_provider[NWPBatchKey.nwp_channel_names][i]
140+
nwp_data_one_channel = nwp_data[b, :, i]
141+
time = nwp_provider[NWPBatchKey.nwp_target_time_utc][b]
142+
time = pd.to_datetime(time, unit="s")
143+
fig.add_trace(
144+
go.Scatter(x=time, y=nwp_data_one_channel, mode="lines", name=channel)
145+
)
146+
147+
fig.update_layout(
148+
title=f"{provider} NWP - example {b}", xaxis_title="Time", yaxis_title="Value"
149+
)
150+
# fig.show(renderer='browser')
151+
name = f"nwp/{provider}_nwp_{b}.png"
152+
fig.write_image(f"{folder}/{name}")
153+
print(f"![](./{name})", file=f)
154+
print("\n", file=f)
155+
156+
for key in keys:
157+
print("\n", file=f)
158+
print(f"#### {key.name}", file=f)
159+
value = nwp_provider[key]
160+
161+
if "time" in key.name:
162+
163+
# make a table with example, shape, max, min
164+
print("| Example | Shape | Max | Min |", file=f)
165+
print("| --- | --- | --- | --- |", file=f)
166+
167+
for example_id in range(n_examples):
168+
value_ts = pd.to_datetime(value[example_id], unit="s")
169+
print(
170+
f"| {example_id} | {len(value_ts)} "
171+
f"| {value_ts.max()} | {value_ts.min()} |",
172+
file=f,
173+
)
174+
175+
elif "channel" in key.name:
176+
177+
# create a table with the channel names with max, min, mean and std
178+
print("| Channel | Max | Min | Mean | Std |", file=f)
179+
print("| --- | --- | --- | --- | --- |", file=f)
180+
for i in range(len(value)):
181+
channel = value[i]
182+
data = nwp_data[:, :, i]
183+
print(
184+
f"| {channel} "
185+
f"| {data.max().item():.2f} "
186+
f"| {data.min().item():.2f} "
187+
f"| {data.mean().item():.2f} "
188+
f"| {data.std().item():.2f} |",
189+
file=f,
190+
)
191+
192+
print(f"Shape={value.shape}", file=f)
193+
194+
elif isinstance(value, torch.Tensor):
195+
print(f"Shape {value.shape=}", file=f)
196+
print(f"Max {value.max():.2f}", file=f)
197+
print(f"Min {value.min():.2f}", file=f)
198+
elif isinstance(value, int):
199+
print(f"{value}", file=f)
200+
else:
201+
print(f"{value}", file=f)
202+
203+
# Satellite
204+
print("## Satellite \n", file=f)
205+
keys = [
206+
BatchKey.satellite_actual,
207+
BatchKey.satellite_t0_idx,
208+
BatchKey.satellite_time_utc,
209+
BatchKey.satellite_time_utc,
210+
BatchKey.satellite_x_geostationary,
211+
BatchKey.satellite_y_geostationary,
212+
]
213+
214+
for key in keys:
215+
216+
print("\n", file=f)
217+
print(f"#### {key.name}", file=f)
218+
value = batch[key]
219+
220+
if "satellite_actual" in key.name:
221+
222+
print(value.shape, file=f)
223+
224+
# average of lat and lon
225+
value = value.mean(dim=(3, 4))
226+
227+
n_examples = value.shape[0]
228+
if limit_examples is not None:
229+
n_examples = min(n_examples, limit_examples)
230+
231+
for b in range(n_examples):
232+
233+
fig = go.Figure()
234+
for i in range(value.shape[2]):
235+
satellite_data_one_channel = value[b, :, i]
236+
time = batch[BatchKey.satellite_time_utc][b]
237+
time = pd.to_datetime(time, unit="s")
238+
fig.add_trace(
239+
go.Scatter(x=time, y=satellite_data_one_channel, mode="lines")
240+
)
241+
242+
fig.update_layout(
243+
title=f"Satellite - example {b}", xaxis_title="Time", yaxis_title="Value"
244+
)
245+
# fig.show(renderer='browser')
246+
name = f"satellite/satellite_{b}.png"
247+
fig.write_image(f"{folder}/{name}")
248+
print(f"![](./{name})", file=f)
249+
print("\n", file=f)
250+
251+
elif "time" in key.name:
252+
253+
# make a table with example, shape, max, min
254+
print("| Example | Shape | Max | Min |", file=f)
255+
print("| --- | --- | --- | --- |", file=f)
256+
257+
for example_id in range(n_examples):
258+
value_ts = pd.to_datetime(value[example_id], unit="s")
259+
print(
260+
f"| {example_id} | {len(value_ts)} "
261+
f"| {value_ts.max()} | {value_ts.min()} |",
262+
file=f,
263+
)
264+
265+
elif "channel" in key.name:
266+
267+
# create a table with the channel names with max, min, mean and std
268+
print("| Channel | Max | Min | Mean | Std |", file=f)
269+
print("| --- | --- | --- | --- | --- |", file=f)
270+
for i in range(len(value)):
271+
channel = value[i]
272+
data = nwp_data[:, :, i]
273+
print(
274+
f"| {channel} "
275+
f"| {data.max().item():.2f} "
276+
f"| {data.min().item():.2f} "
277+
f"| {data.mean().item():.2f} "
278+
f"| {data.std().item():.2f} |",
279+
file=f,
280+
)
281+
282+
print(f"Shape={value.shape}", file=f)
283+
284+
elif isinstance(value, torch.Tensor):
285+
print(f"Shape {value.shape=}", file=f)
286+
print(f"Max {value.max():.2f}", file=f)
287+
print(f"Min {value.min():.2f}", file=f)
288+
elif isinstance(value, int):
289+
print(f"{value}", file=f)
290+
else:
291+
print(f"{value}", file=f)
292+
293+
294+
# For example you can run it like this
295+
# with open("batch.md", "w") as f:
296+
# sys.stdout = f
297+
# d = torch.load("000000.pt")
298+
# visualise_batch(d)

‎ocf_datapipes/visualization/batch.py

Lines changed: 0 additions & 1 deletion
This file was deleted.

0 commit comments

Comments
 (0)