Embedded Files
Learn more about the data avilable on GridLive.
This page provides overviews and example figures of the data available from GridLive's /smart_meter/ and /esa_metadata/ end points, along with the code used to generate the example figures.
Smart meter time-series end points
Smart meter time-series end points
The /smart_meter/ end point provides access to time-series smart meter energy consumption data. For each reading, the end point returns the time the reading was taken at, the ID of electricity supply area to associate it with metadata, the re/active device count, re/active primary consumption, and re/active total consumption for 8 columns total.
This figure illustrates the different amounts and duration of data provided by the GridLive /smart_meter/ end points by plotting the active primary consumption.
# Script to illustrate the different smart_meter endpoints
# Standard library imports
from datetime import datetime, timedelta
# External library imports
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
import pandas as pd
from requests import get
# Dictionary of end point, specifying the duration and any other parameters
valid_end_points = {
"/smart_meter": (timedelta(hours=2), {}),
"/smart_meter/dno/NPG": (timedelta(hours=24), {}),
"/smart_meter/license_area/South Wales": (timedelta(hours=24), {}),
"/smart_meter/in/SP08": (timedelta(hours=24), {}), # Birmingham
"/smart_meter/near/NZ26": (timedelta(hours=24), {"radius": 1_000}), # Newcastle
"/smart_meter/esa/1325075231911461884": (timedelta(days=365), {}),
}
# Fetch data from this point in time onwards
time_from = datetime(2024, 6, 1, 0, 0, 0)
# Setup for plots
fig, axes = plt.subplots(2, 3, figsize=(12, 6))
axes = axes.flatten()
for ii, (end_point, (duration, params2)) in enumerate(valid_end_points.items()):
print(f"Fetching from {end_point}...")
response = get(
f"https://api.gridlive.shef.ac.uk/{end_point}",
headers={"Authorization": "YOUR API KEY HERE"},
params={
"limit": 0,
"start_datetime": time_from.isoformat(),
"end_datetime": (time_from + duration).isoformat(),
}
| params2,
)
assert response.status_code == 200
# Convert the data to a pandas DataFrame
df = pd.DataFrame(response.json())
assert len(df) > 0
df["data_timestamp"] = pd.to_datetime(df["data_timestamp"])
df = (
df.drop(
columns=[
"active_device_count",
"active_secondary_consumption_import",
"active_total_consumption_import",
"reactive_device_count",
"reactive_total_consumption_import",
],
)
.rename(
columns={
"data_timestamp": "time",
"active_primary_consumption_import": "use",
},
)
.dropna()
.sort_values(by="time")
)
df.loc[df["use"] > 1e5, "use"] = float("nan")
df = df.set_index(keys=["time", "esa_id"]).unstack()
# Plot
axes[ii].plot(df.index, df.to_numpy() / 1000)
axes[ii].set_xlabel("Time & Date") if ii > 2 else axes[ii].set_xlabel("")
axes[ii].set_ylabel("Consumption (mWh)") if ii % 3 == 0 else axes[ii].set_ylabel("")
axes[ii].set_title(end_point)
s = f" {df.shape[1]} series of {len(df)} data points"
axes[ii].text(0, 0.93, s, transform=axes[ii].transAxes, style="italic")
axes[ii].set_xticks([time_from + ii * duration / 2 for ii in range(0, 3)])
axes[ii].xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m-%d %H:%M"))
# Save plot
plt.tight_layout()
plt.savefig("smart_meter_examples.png", dpi=300)
# To convert to jpeg use ImageMagick to run:
# convert smart_meter_examples.png -quality 80 smart_meter_examples.jpg
ESA Metadata end points
ESA Metadata end points
The /esa_metadata/ end point provides access to information about the electricity supply areas (ESAa) for which there are time-series measurements. For each ESA, the end point returns the ID of ESA to associate it with time-series data, the license area to which the ESA belongs, the Distribution Network Operator to which the ESA belongs, the ESA level (e.g., LV feeder), the location as UK Ordnance Survey Grid Reference eastings and northings, a unique ID string, the secondary substation ID and name, the LV feeder ID and name, and when data for this ESA was last updated in the database. In total there are 12 columns.
This figure illustrates the ESAs available and how different ESAs can be selected from the GridLive /esa_metadata/ end points.
# An example of fetching metadata from GridLive API and plotting it on a map using Plotly
# Note, requires Chrome in order to save figure, see https://github.com/plotly/Kaleido
# Standard library imports
import io
import warnings
# External library imports
import geojson
import geopandas as gpd
import numpy as np
import pandas as pd
import plotly.express as px
import requests
from os_grid_reference.ngr_decode import ngr_decode
from plotly.subplots import make_subplots
from pyproj import Transformer
# Suppress plotly deprecation warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
BASE_URL = "https://api.gridlive.shef.ac.uk/"
# Your GridLive API key
API_KEY = "YOUR API KEY HERE"
HEADERS = {"Authorization": API_KEY}
# Get DNO license areas polygon
print("Fetching DNO license areas...")
dno_license_areas = gpd.read_file(
"https://api.neso.energy/dataset/0e377f16-95e9-4c15-a1fc-49e06a39cfa0/resource/e96db306-aaa8-45be-aecd-65b34d38923a/download/dno_license_areas_20200506.geojson"
).to_crs("EPSG:4326")
dno_license_areas = dno_license_areas.__geo_interface__
dno_license_areas_layer = {
"source": dno_license_areas,
"type": "line",
"color": "black",
"line": {"width": 0.5},
}
print("DNO license areas fetched \n")
# Create a circle polygon around a point
earth_radius_in_km = 6371.0
circle_radius = 50.0
circle_center_latitude = 51.4816
circle_center_longitude = -3.1791
angles = np.linspace(0, 2 * np.pi, 100)
# Angular distance in radians
rad_dist = circle_radius / earth_radius_in_km
points = []
for angle in angles:
lat_rad = np.radians(circle_center_latitude)
lon_rad = np.radians(circle_center_longitude)
lat_point = np.arcsin(
np.sin(lat_rad) * np.cos(rad_dist)
+ np.cos(lat_rad) * np.sin(rad_dist) * np.cos(angle)
)
lon_point = lon_rad + np.arctan2(
np.sin(angle) * np.sin(rad_dist) * np.cos(lat_rad),
np.cos(rad_dist) - np.sin(lat_rad) * np.sin(lat_point),
)
points.append((np.degrees(lon_point), np.degrees(lat_point)))
# Close the polygon by repeating the first point at the end
points.append(points[0])
circle = geojson.Polygon([points])
circle_layer = {
"source": circle,
"type": "line",
"color": "rgba(255, 0, 0, 0.3)",
}
# Create a square polygon to represent OS grid reference SE
tile = ngr_decode("SE")
sw = (tile.sw_easting, tile.sw_northing)
ne = (tile.ne_easting, tile.ne_northing)
se = (sw[0] + tile.size, sw[1])
nw = (ne[0] - tile.size, ne[1])
coordinates = [
sw,
se,
ne,
nw,
sw, # close polygon back to SW
]
# Convert OSGB36 (EPSG:27700) to WGS84 (EPSG:4326)
transformer = Transformer.from_crs("EPSG:27700", "EPSG:4326", always_xy=True)
coordinates = [
transformer.transform(easting, northing) for easting, northing in coordinates
]
square = geojson.Polygon([coordinates])
square_layer = {
"source": square,
"type": "line",
"color": "rgba(255, 0, 0, 0.3)",
}
# Fetch metadata from GridLive API
# Specify map view for each endpoint
endpoints = [
{
"name": "/",
"center": {"lat": 54.5, "lon": -2},
"zoom": 4.8,
"layers": [dno_license_areas_layer],
},
{
"name": "/license_area/West%20Midlands",
"center": {"lat": 54.5, "lon": -2},
"zoom": 4.8,
"layers": [dno_license_areas_layer],
},
{
"name": "/dno/UKPN",
"center": {"lat": 54.5, "lon": -2},
"zoom": 4.8,
"layers": [dno_license_areas_layer],
},
{
"name": "/esa/1241887139609918156",
"center": {"lat": 52.15, "lon": -3.38},
"zoom": 6,
"layers": [],
},
{
"name": "/in/SE",
"center": {"lat": 53.9, "lon": -1.08},
"zoom": 6,
"layers": [square_layer],
},
{
"name": f"/near/ST184763?radius={circle_radius * 1000}",
"center": {"lat": circle_center_latitude, "lon": circle_center_longitude},
"zoom": 6,
"layers": [circle_layer],
},
]
print("Fetching metadata from GridLive API...")
for idx, endpoint in enumerate(endpoints):
url = f"{BASE_URL}/esa_metadata{endpoint['name']}"
params = {"limit": 0}
res = requests.get(url, params=params, headers=HEADERS)
esas = pd.read_json(io.StringIO(res.text))
esas = gpd.GeoDataFrame(
esas,
geometry=gpd.points_from_xy(
esas["esa_location_eastings"],
esas["esa_location_northings"],
),
crs="EPSG:27700",
)
esas = esas.to_crs("EPSG:4326")
esas["longitude"] = esas.geometry.x
esas["latitude"] = esas.geometry.y
print(f"Endpoint: {endpoint['name']}, Number of ESAs: {len(esas)}")
endpoints[idx]["data"] = esas
print("Metadata fetched \n")
# Create subplots of maps using Plotly
fig = make_subplots(
rows=2,
cols=3,
subplot_titles=[
f"{endpoint['name']} ({len(endpoint['data'])} ESAs)" for endpoint in endpoints
],
vertical_spacing=0.1,
specs=[[{"type": "mapbox"}] * 3, [{"type": "mapbox"}] * 3],
)
# Plot data
for idx, endpoint in enumerate(endpoints):
row = idx // 3 + 1
col = idx % 3 + 1
map_styles = {
"style": "carto-positron",
"center": endpoint["center"],
"zoom": endpoint["zoom"],
"layers": endpoint["layers"],
}
subplot = px.scatter_mapbox(
endpoint["data"],
lat="latitude",
lon="longitude",
hover_name="dno_name",
)
traces = subplot.data
for trace in traces:
fig.add_trace(trace, row=row, col=col)
fig.update_layout({f"mapbox{idx + 1}": map_styles})
fig = fig.update_layout(
title_text="ESA Metadata from Multiple Endpoints",
showlegend=False,
)
print("Saving figure...")
filename = f"esa_metadata.png"
fig.write_image(filename, height=1800, width=1400, scale=2)
# fig.write_html(f"{filename}.html")
print(f"Figure saved to {filename}")
Report abuse