📘 Lesson 4: GRDC Data

📘 Lesson 4: GRDC Data#

lesson_number = 4
print(f'lesson{lesson_number}')

lesson4

🎯 Learning Objectives#

  • Read data from the GRDC database.

Prepare Google Colab Environment#

#@title Connect to Google Drive {display-mode:"form"}
CONNECT_TO_DRIVE = False #@param {type:"boolean"}

import os

if CONNECT_TO_DRIVE:
    from google.colab import drive
    # Mount Google Drive
    drive.mount('/content/drive')

    # Define the desired working directory path
    working_dir = '/content/drive/MyDrive/hello-pypsa'

    # Create the directory if it doesn't exist
    if not os.path.exists(working_dir):
        os.makedirs(working_dir)
        print(f"Directory '{working_dir}' created.")
    else:
        print(f"Directory '{working_dir}' already exists.")

    # Change the current working directory
    os.chdir(working_dir)

    print(f"Current working directory: {os.getcwd()}")
else:
    print("Not connecting to Google Drive.")

import os

#@title Install Packages {display-mode:"form"}
INSTALL_PACKAGES = False #@param {type:"boolean"}

# Check if packages have already been installed in this session to prevent re-installation
if INSTALL_PACKAGES and not os.environ.get('PYPSA_PACKAGES_INSTALLED'):
  !pip install pypsa pypsa[excel] folium mapclassify cartopy
  !pip install git+https://github.com/PriyeshGosai/pypsa_network_viewer.git
  os.environ['PYPSA_PACKAGES_INSTALLED'] = 'true'
elif not INSTALL_PACKAGES:
  print("Skipping package installation.")
else:
  print("PyPSA packages are already installed for this session.")

#@title Download the file for this notebook {display-mode:"form"}
DOWNLOAD_FILE = False #@param {type:"boolean"}

if DOWNLOAD_FILE:
    !gdown "https://drive.google.com/uc?id=1My8j2qRcjjhVhC5bL657oYTk7-OKDxkE"


else:
    print("Skipping file download.")

# !pip install git+https://github.com/pypsa/pypsa

📄 Case Study#

You have been provided

  • The file data/power_pool_data/rucana/GRDC-Daily.nc that was downloaded for the Rucana measurement station from the GRDC data poral.

  • Read the data, plot the flows and create a table for all relevant climate years.

Explore Data#

import xarray as xr
import numpy as np
import pandas as pd

file_name = "data/power_pool_data/rucana/GRDC-Daily.nc"

ds = xr.open_dataset(file_name, decode_times=False)

time_raw = ds.time.values

# Convert time to datetime64
time_converted = np.array(time_raw, dtype="timedelta64[D]") + np.datetime64("1700-01-01")

runoff = ds.runoff_mean.isel(id=0).values
runoff

# Create daily time series DataFrame
runoff_df = pd.DataFrame({
    "date": time_converted,
    "runoff_m3s": runoff
}).set_index("date")

runoff_df.plot()

Functions#

import xarray as xr
import pandas as pd
import numpy as np

def extract_hourly_runoff_by_year(filename, missing_threshold=24):
    """
    Processes a GRDC NetCDF file and returns a pivoted, cleaned DataFrame of hourly runoff.
    
    Parameters:
        filename (str): Path to the NetCDF file.
        missing_threshold (int): Maximum number of NaNs allowed per year.

    Returns:
        pd.DataFrame: DataFrame indexed by hour of year, with each column representing a year.
    """
    # Load dataset with decode_times=False to manually handle time
    ds = xr.open_dataset(filename, decode_times=False)
    
    # Extract variables
    runoff = ds["runoff_mean"].isel(id=0).values
    time_raw = ds["time"].values
    runoff = np.where(runoff == -999.0, np.nan, runoff)

    # Convert time to datetime64
    time_converted = np.array(time_raw, dtype="timedelta64[D]") + np.datetime64("1700-01-01")

    # Create daily time series DataFrame
    runoff_df = pd.DataFrame({
        "date": time_converted,
        "runoff_m3s": runoff
    }).set_index("date")

    # Resample to hourly and apply forward fill
    runoff_hourly = runoff_df.resample("h").ffill()

    # Smooth with centered 24-hour moving average and fill edges
    runoff_hourly = runoff_hourly.rolling(window=24, center=True, win_type="boxcar").mean()
    runoff_hourly = runoff_hourly.ffill().bfill()

    # Drop leap days
    runoff_hourly = runoff_hourly[~((runoff_hourly.index.month == 2) & (runoff_hourly.index.day == 29))]

    # Add year and hour-of-year
    df = runoff_hourly.copy()
    df["year"] = df.index.year
    df["hour_of_year"] = ((df.index.dayofyear - 1) * 24) + df.index.hour

    # Pivot to [hour_of_year x year]
    pivot = df.pivot(index="hour_of_year", columns="year", values="runoff_m3s")

    # Drop years with more than `missing_threshold` NaNs
    pivot = pivot.dropna(axis=1, thresh=pivot.shape[0] - missing_threshold)

    # Fill remaining missing data
    pivot = pivot.ffill().bfill()

    # Drop last 24 hours to standardize length
    return pivot.iloc[:-23]

def check_runoff_data_cleanliness(df):
    """
    Checks basic data quality metrics for a pivoted runoff dataset.
    
    Parameters:
        df (pd.DataFrame): DataFrame with hour_of_year as index and years as columns.
    """
    print("✅ DataFrame Shape:", df.shape)
    print()

    # Missing data check
    total_nans = df.isna().sum().sum()
    print("🔍 Total missing values:", total_nans)

    if total_nans > 0:
        print("⚠️ Missing values per year (top 5):")
        print(df.isna().sum().sort_values(ascending=False).head())
        print()
    else:
        print("✅ No missing values.")
        print()

    # Expected hourly rows (after dropping leap day and last day)
    expected_hours = 8760 - 24 - 24
    if df.shape[0] != expected_hours:
        print(f"⚠️ Unexpected number of rows: {df.shape[0]} (expected: {expected_hours})")
    else:
        print(f"📆 All years span expected {expected_hours} hourly steps.")

    print("📅 Year range:", df.columns.min(), "to", df.columns.max())
    print("📈 Value range (runoff in m³/s):")
    print("Min:", df.min().min())
    print("Max:", df.max().max())
    print("Mean:", df.mean().mean())

View and extract data#

file_path = "data/power_pool_data/rucana/GRDC-Daily.nc"
output_path = 'data/power_pool_data/rucana/rucana.csv'

df_river = extract_hourly_runoff_by_year(file_path)

check_runoff_data_cleanliness(df_river)

import plotly.graph_objects as go

# Create a Plotly figure
fig = go.Figure()

# Add one line per year
for year in df_river.columns:
    fig.add_trace(go.Scatter(
        x=df_river.index,
        y=df_river[year],
        mode='lines',
        name=str(year),
        line=dict(width=1)
    ))

# Update layout
fig.update_layout(
    title="Hourly Runoff by Year",
    xaxis_title="Hour of Year",
    yaxis_title="Runoff (m³/s)",
    template="plotly_white",
    height=600,
    width=1000,
    showlegend=False  # Set True if legend is desired
)

fig.show()

df_river.to_csv(output_path)