Fix Python 3.12 / NumPy 2.0 / pandas 2.x compatibility and runtime bugs

pytest/test_risktools.py

@@ -1,11 +1,12 @@
-from numpy.linalg.linalg import eigvals
 import pandas as pd
 import numpy as np
 import os
 import json
 import sys
+import socket
 import plotly.graph_objects as go
 import time
+import pytest
 import yfinance as yf
 
 sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../src/")
@@ -40,6 +41,24 @@
 ms = dict(username=os.getenv("MS_USER"), password=os.getenv("MS_PASS"))
 
 
+def _network_available(host="8.8.8.8", port=53, timeout=2):
+    """Return True if a network connection can be established."""
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.settimeout(timeout)
+        s.connect((host, port))
+        s.close()
+        return True
+    except OSError:
+        return False
+
+
+requires_network = pytest.mark.skipif(
+    not _network_available(), reason="Network not available"
+)
+
+
+
 def _load_json(fn, dataframe=True):
     path = os.path.dirname(__file__)
     fp = os.path.join(path, fn)
@@ -108,6 +127,7 @@ def test_get_prices():
     #     i += 1
 
 
+@requires_network
 def test_ir_df_us():
 
     df = _load_json("./data/ir_df_us.json")
@@ -151,6 +171,7 @@ def test_bond():
     assert round(bo, 4) == 0.9878, "bond Test 3 failed"
 
 
+@requires_network
 def test_trade_stats():
 
     # df = data.DataReader(["SPY", "AAPL"], "yahoo", "2000-01-01", "2012-01-01")
@@ -323,7 +344,7 @@ def test_prompt_beta():
 
     x = rt.returns(df=dfwide, ret_type="abs", period_return=1)
     x = rt.roll_adjust(df=x, commodity_name="cmewti", roll_type="Last_Trade")
-    x = x[~x.index.isin(["2020-04-20", "2020-04-21"])]
+    x = x[~x.index.isin(pd.to_datetime(["2020-04-20", "2020-04-21"]))]
     x = x.loc['2010-01-04':'2022-12-30',:]
 
     ts = (
@@ -442,6 +463,7 @@ def test_stl_decomposition():
     pass
 
 
+@requires_network
 def test_get_eia_df():
     ts = rt.get_eia_df("PET.MCRFPTX2.M", key=up["eia"])
 
@@ -489,11 +511,13 @@ def test_chart_zscore():
     assert isinstance(stl, go.Figure), "chart_zscore Test failed"
 
 
+@requires_network
 def test_chart_eia_sd():
     fig = rt.chart_eia_sd("mogas", up["eia"])
     assert isinstance(fig, go.Figure), "chart_eia_sd Test failed"
 
 
+@requires_network
 def test_chart_eia_steo():
     fig = rt.chart_eia_steo(up["eia"])
     assert isinstance(fig, go.Figure), "chart_eia_steo Test failed"

src/risktools/_charts.py

@@ -180,6 +180,8 @@ def chart_eia_sd(market, key, start_dt="2010-01-01", output="chart", **kwargs):
     eia = eia[eia.sd_category == market]
 
     df = get_eia_df(eia.tick_eia.to_list(), key=key)
+    if df.empty:
+        raise ValueError("No EIA data returned. Check your API key and network connection.")
     df = df.merge(
         eia[["tick_eia", "category"]], left_on=["series_id"], right_on=["tick_eia"]
     ).drop("tick_eia", axis=1)
@@ -374,6 +376,8 @@ def chart_eia_steo(key, start_dt=None, market="globalOil", output="chart", **kwa
         }
 
     df = get_eia_df(list(tickers.keys()), key=key)
+    if df.empty:
+        raise ValueError("No EIA data returned. Check your API key and network connection.")
     df["name"] = df["series_id"].map(tickers)
     df = (
         df[["date", "value", "name"]]

src/risktools/_main_functions.py

@@ -79,7 +79,7 @@ def ir_df_us(quandl_key=None, ir_sens=0.01, date=None):
     x.columns = ["maturity", "yield"]
     x["index"] = x["maturity"]
     x["yield"] /= 100
-    x["maturity"] = x.maturity.str.extract("(\d+)").astype("float")
+    x["maturity"] = x.maturity.str.extract(r"(\d+)").astype("float")
 
     # change maturity numbers to year fraction for first four rows
     x.iloc[1:4, x.columns.get_loc("maturity")] /= 12.0
@@ -219,13 +219,14 @@ def trade_stats(R, Rf=0):
 
         # need to dropna to calc perc_win properly
         con_clean = con.dropna()
+        n_nonzero = con_clean[con_clean != 0].shape[0]
         rs[lab]["perc_win"] = (
-            con_clean[con_clean > 0].shape[0] / con_clean[con_clean != 0].shape[0]
+            con_clean[con_clean > 0].shape[0] / n_nonzero if n_nonzero > 0 else _np.nan
         )
-        rs[lab]["perc_in_mkt"] = con_clean[con_clean != 0].shape[0] / con_clean.shape[0]
+        rs[lab]["perc_in_mkt"] = n_nonzero / con_clean.shape[0] if con_clean.shape[0] > 0 else _np.nan
 
-        rs[lab]["dd_length"] = max(y["length"])
-        rs[lab]["dd_max"] = min(y["return"])
+        rs[lab]["dd_length"] = max(y["length"]) if len(y["length"]) > 0 else _np.nan
+        rs[lab]["dd_max"] = min(y["return"]) if len(y["return"]) > 0 else _np.nan
 
     if series_flag == True:
         rs = rs["trade_stats"]
@@ -465,7 +466,7 @@ def garch(df, out="data", scale=None, show_fig=True, forecast_horizon=1, **kwarg
 
 
 def prompt_beta(df, period="all", beta_type="all", output="chart"):
-    """
+    r"""
     Returns array/dataframe of betas for futures contract returns of a commodity
     with it's front contract (i.e. next most expirying contract). For use with futures
     contracts (i.e. NYMEX WTI: CL01, CL02, CL03 and so forth) with standardized expiry periods.
@@ -940,16 +941,17 @@ def _get_eia_df_v2(tables, key, sleep):
                 tmp["response"]["data"],
                 columns=["period", "series-description", "value"],
             )
-        except:
-            print(f"Error in table {tbl}")
-            print(r.text)
+        except Exception as e:
+            print(f"Error in table {tbl}: {e}")
             continue
         tf["series_id"] = tbl
         eia = _pd.concat([eia, tf], axis=0)
         time.sleep(sleep)
         # eia = eia.append(tf)
 
     eia = eia.rename(columns={"period": "date", "series-description": "table_name"})
+    if eia.empty:
+        return _pd.DataFrame(columns=["date", "value", "table_name", "series_id"])
     eia.loc[eia.date.str.len() < 7, "date"] += "01"
     eia.date = _pd.to_datetime(eia.date)
     return eia[["date", "value", "table_name", "series_id"]]

src/risktools/_multivariate.py

@@ -130,11 +130,11 @@ def generate_eps_MV(cor, T, dt, sims=1000, mu=None, seed=None):
 
     # if ~isinstance(sigma, _np.ndarray):
     #     sigma = _np.array(sigma)
-    if ~isinstance(cor, _np.ndarray):
+    if not isinstance(cor, _np.ndarray):
         cor = _np.array(cor)
 
     if mu is not None:
-        if ~isinstance(mu, _np.ndarray):
+        if not isinstance(mu, _np.ndarray):
             mu = _np.array(mu)
     else:
         mu = _np.zeros(cor.shape[0])
@@ -204,17 +204,17 @@ def simGBM_MV(s0, r, sigma, T, dt, mu=None, cor=None, eps=None, sims=1000, seed=
     if (cor is None) & (eps is None):
         raise ValueError("correlation matrix cor required if eps not passed")
 
-    if ~isinstance(s0, _np.ndarray):
+    if not isinstance(s0, _np.ndarray):
         s0 = _np.array(s0)
-    if ~isinstance(sigma, _np.ndarray):
+    if not isinstance(sigma, _np.ndarray):
         sigma = _np.array(sigma)
-    if ~isinstance(r, _np.ndarray):
+    if not isinstance(r, _np.ndarray):
         r = _np.array(r)
-    if ~isinstance(cor, _np.matrix):
+    if not isinstance(cor, _np.matrix):
         cor = _np.matrix(cor)
 
     if mu is not None:
-        if ~isinstance(mu, _np.ndarray):
+        if not isinstance(mu, _np.ndarray):
             mu = _np.array(mu)
     else:
         mu = _np.zeros(len(s0))

src/risktools/_pa.py

@@ -6,7 +6,7 @@
 
 
 def return_cumulative(r, geometric=True):
-    """
+    r"""
     Based on the function Return.annualize from the R package PerformanceAnalytics
     by Peter Carl and Brian G. Peterson
 
@@ -52,7 +52,7 @@ def return_cumulative(r, geometric=True):
 
 
 def return_annualized(r, scale=None, geometric=True):
-    """
+    r"""
     Based on the function Return.annualize from the R package PerformanceAnalytics
     by Peter Carl and Brian G. Peterson
 
@@ -115,6 +115,9 @@ def return_annualized(r, scale=None, geometric=True):
     r = r.dropna()
     n = r.shape[0]
 
+    if n == 0:
+        return _np.nan
+
     if geometric:
         res = (r.add(1).cumprod() ** (scale / n) - 1).iloc[-1]
     else:
@@ -123,7 +126,7 @@ def return_annualized(r, scale=None, geometric=True):
 
 
 def return_excess(R, Rf=0):
-    """
+    r"""
     Calculates the returns of an asset in excess of the given risk free rate
 
     Calculates the returns of an asset in excess of the given "risk free rate"
@@ -176,7 +179,7 @@ def return_excess(R, Rf=0):
 
 
 def sd_annualized(x, scale=None, *args):
-    """
+    r"""
     calculate a multiperiod or annualized Standard Deviation
 
     Standard Deviation of a set of observations \eqn{R_{a}} is given by:
@@ -232,7 +235,7 @@ def sd_annualized(x, scale=None, *args):
     >>> rt.sd_annualized(x=df[('Adj Close','SPY')])
     >>> rt.sd_annualized(x=df['Adj Close'])
     """
-    if (~isinstance(x, _pd.DataFrame) & ~isinstance(x, _pd.Series)) == True:
+    if (not isinstance(x, _pd.DataFrame) and not isinstance(x, _pd.Series)):
         raise ValueError("x must be a pandas Series or DataFrame")
 
     if isinstance(x.index, _pd.DatetimeIndex):
@@ -263,7 +266,7 @@ def sd_annualized(x, scale=None, *args):
 
 
 def omega_sharpe_ratio(R, MAR, *args):
-    """
+    r"""
     Omega-Sharpe ratio of the return distribution
 
     The Omega-Sharpe ratio is a conversion of the omega ratio to a ranking statistic
@@ -301,11 +304,11 @@ def omega_sharpe_ratio(R, MAR, *args):
         if isinstance(R.index, _pd.DatetimeIndex) & isinstance(
             MAR, (_pd.Series, _pd.DataFrame)
         ):
-            if ~isinstance(MAR.index, _pd.DatetimeIndex):
+            if not isinstance(MAR.index, _pd.DatetimeIndex):
                 raise ValueError(
                     "MAR index must be a datatime index if MAR and R are a Dataframe or Series with a datetime index"
                 )
-        elif ~isinstance(R.index, _pd.DatetimeIndex) & isinstance(
+        elif (not isinstance(R.index, _pd.DatetimeIndex)) and isinstance(
             MAR, (_pd.Series, _pd.DataFrame)
         ):
             if isinstance(MAR.index, _pd.DatetimeIndex):
@@ -336,7 +339,7 @@ def omega_sharpe_ratio(R, MAR, *args):
 
 
 def upside_risk(R, MAR=0, method="full", stat="risk"):
-    """
+    r"""
     upside risk, variance and potential of the return distribution
 
     Upside Risk is the similar of semideviation taking the return above the
@@ -403,11 +406,11 @@ def upside_risk(R, MAR=0, method="full", stat="risk"):
         if isinstance(R.index, _pd.DatetimeIndex) & isinstance(
             MAR, (_pd.Series, _pd.DataFrame)
         ):
-            if ~isinstance(MAR.index, _pd.DatetimeIndex):
+            if not isinstance(MAR.index, _pd.DatetimeIndex):
                 raise ValueError(
                     "MAR index must be a datatime index if MAR and R are a Dataframe or Series with a datetime index"
                 )
-        elif ~isinstance(R.index, _pd.DatetimeIndex) & isinstance(
+        elif (not isinstance(R.index, _pd.DatetimeIndex)) and isinstance(
             MAR, (_pd.Series, _pd.DataFrame)
         ):
             if isinstance(MAR.index, _pd.DatetimeIndex):
@@ -489,11 +492,11 @@ def downside_deviation(R, MAR=0, method="full", potential=False):
         if isinstance(R.index, _pd.DatetimeIndex) & isinstance(
             MAR, (_pd.Series, _pd.DataFrame)
         ):
-            if ~isinstance(MAR.index, _pd.DatetimeIndex):
+            if not isinstance(MAR.index, _pd.DatetimeIndex):
                 raise ValueError(
                     "MAR index must be a datatime index if MAR and R are a Dataframe or Series with a datetime index"
                 )
-        elif ~isinstance(R.index, _pd.DatetimeIndex) & isinstance(
+        elif (not isinstance(R.index, _pd.DatetimeIndex)) and isinstance(
             MAR, (_pd.Series, _pd.DataFrame)
         ):
             if isinstance(MAR.index, _pd.DatetimeIndex):
@@ -531,7 +534,7 @@ def downside_deviation(R, MAR=0, method="full", potential=False):
 
 
 def sharpe_ratio_annualized(R, Rf=0, scale=None, geometric=True):
-    """
+    r"""
     calculate annualized Sharpe Ratio
 
     The Sharpe Ratio is a risk-adjusted measure of return that uses standard
@@ -619,7 +622,7 @@ def drawdowns(R, geometric=True):
 
 
 def find_drawdowns(R, geometric=True, *args):
-    """
+    r"""
     Find the drawdowns and drawdown levels in a timeseries.
 
     find_drawdowns() will find the starting period, the ending period, and
@@ -691,16 +694,21 @@ def find_drawdowns(R, geometric=True, *args):
         rs[lab]["to"] = _np.array([]).astype(int)
         rs[lab]["length"] = _np.array([]).astype(int)
         rs[lab]["trough"] = _np.array([]).astype(int)
+        rs[lab]["peaktotrough"] = _np.array([]).astype(int)
+        rs[lab]["recovery"] = _np.array([]).astype(int)
 
-        if con[0] >= 0:
+        if con.empty:
+            continue
+
+        if con.iloc[0] >= 0:
             prior_sign = 1
         else:
             prior_sign = 0
 
         frm = 0
         to = 0
         dmin = 0
-        sofar = con[0]
+        sofar = con.iloc[0]
 
         for i, r in enumerate(con):  # .iteritems():
             if r < 0:
@@ -734,10 +742,10 @@ def find_drawdowns(R, geometric=True, *args):
         rs[lab]["peaktotrough"] = rs[lab]["trough"] - rs[lab]["from"] + 1
         rs[lab]["recovery"] = rs[lab]["to"] - rs[lab]["trough"]
 
-        # if original parameter was a series, remove top layer of
-        # results dictionary
-        if series_flag == True:
-            rs = rs["drawdown"]
+    # if original parameter was a series, remove top layer of
+    # results dictionary
+    if series_flag == True:
+        rs = rs["drawdown"]
 
     return rs
 
@@ -798,7 +806,7 @@ def _beta(y, x, subset=None):
 
 
 def CAPM_beta(Ra, Rb, Rf=0, kind="all"):
-    """
+    r"""
     calculate single factor model (CAPM) beta
 
     The single factor model or CAPM Beta is the beta of an asset to the variance
@@ -959,7 +967,7 @@ def _check_ts(R, scale, name="R"):
     -------
     tuple with R as Series or Dataframe and scale as int
     """
-    if (~isinstance(R, _pd.DataFrame) & ~isinstance(R, _pd.Series)) == True:
+    if (not isinstance(R, _pd.DataFrame) and not isinstance(R, _pd.Series)):
         raise ValueError(f"{name} must be a pandas Series or DataFrame")
 
     if isinstance(R.index, _pd.DatetimeIndex):