Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit eb1267b

Browse files
Add uniform regularized regression
Modify forward_regression_orthogonal_least_squares.py and estimators.py to enable ridge regression. You need to set ridge_param when calling the FROLS class.
1 parent bd763cf commit eb1267b

File tree

2 files changed

+57
-2
lines changed

2 files changed

+57
-2
lines changed

‎sysidentpy/model_structure_selection/forward_regression_orthogonal_least_squares.py

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ class FROLS(Estimators, BaseMSS):
8888
The convergence coefficient (learning rate) of the filter.
8989
eps : float
9090
Normalization factor of the normalized filters.
91+
ridge_param : float
92+
Regularization prameter used in ridge regression
9193
gama : float, default=0.2
9294
The leakage factor of the Leaky LMS method.
9395
weight : float, default=0.02
@@ -164,6 +166,7 @@ def __init__(
164166
offset_covariance: float = 0.2,
165167
mu: float = 0.01,
166168
eps: np.float64 = np.finfo(np.float64).eps,
169+
ridge_param: np.float64 = np.finfo(np.float64).eps, # default is machine eps
167170
gama: float = 0.2,
168171
weight: float = 0.02,
169172
basis_function: Union[Polynomial, Fourier] = Polynomial(),
@@ -191,6 +194,7 @@ def __init__(
191194
offset_covariance=offset_covariance,
192195
mu=mu,
193196
eps=eps,
197+
ridge_param=ridge_param, # ridge regression parameter
194198
gama=gama,
195199
weight=weight,
196200
basis_function=basis_function,
@@ -294,9 +298,16 @@ def error_reduction_ratio(self, psi, y, process_term_number):
294298
for j in np.arange(i, dimension):
295299
# Add `eps` in the denominator to omit division by zero if
296300
# denominator is zero
301+
# To implement regularized regression (ridge regression), add
302+
# ridgePparam to psi.T @ psi. See S. Chen, Local regularization assisted
303+
# orthogonal least squares regression, Neurocomputing 69 (2006) 559–585.
304+
# The version implemeted below uses the same regularization for every feature,
305+
# What Chen refers to Uniform regularized orthogonal least squares (UROLS)
306+
# Set to tiny (self.eps) when you are not regularizing. ridge_param = eps is
307+
# the default.
297308
tmp_err[j] = (np.dot(tmp_psi[i:, j].T, tmp_y[i:]) ** 2) / (
298-
np.dot(tmp_psi[i:, j].T, tmp_psi[i:, j]) *squared_y+ self.eps
299-
)
309+
(np.dot(tmp_psi[i:, j].T, tmp_psi[i:, j]) + self.ridge_param) *squared_y
310+
)+self.eps
300311

301312
if i == process_term_number:
302313
break

‎sysidentpy/parameter_estimation/estimators.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,13 @@ def __init__(
2323
offset_covariance=0.2,
2424
mu=0.01,
2525
eps=np.finfo(np.float64).eps,
26+
ridge_param = np.finfo(np.float64).eps, # for regularized ridge regression
2627
gama=0.2,
2728
weight=0.02,
2829
basis_function=None,
2930
):
3031
self.eps = eps
32+
self.ridge_param = ridge_param # for regularized ridge regression
3133
self.mu = mu
3234
self.offset_covariance = offset_covariance
3335
self.max_lag = max_lag
@@ -49,6 +51,7 @@ def _validate_params(self):
4951
"offset_covariance": self.offset_covariance,
5052
"mu": self.mu,
5153
"eps": self.eps,
54+
"ridge_param": self.ridge_param, # for regularized ridge regression
5255
"gama": self.gama,
5356
"weight": self.weight,
5457
}
@@ -116,6 +119,47 @@ def least_squares(self, psi, y):
116119
y = y[self.max_lag :, 0].reshape(-1, 1)
117120
theta = np.linalg.lstsq(psi, y, rcond=None)[0]
118121
return theta
122+
123+
def ridge_regression(self, psi, y):
124+
"""Estimate the model parameters using the regularized least squares method
125+
known as ridge regression. Based on the least_squares module and uses
126+
the same data format but you need to pass ridge_param in the call to
127+
FROLS
128+
129+
Parameters
130+
----------
131+
psi : ndarray of floats
132+
The information matrix of the model.
133+
y : array-like of shape = y_training
134+
The data used to training the model.
135+
ridge_param : ridge regression parameter that regularizes the algorithm
136+
to prevent over fitting. If the input is a noisy signal, the ridge
137+
parameter is likely to be set close to the noise level, at least
138+
as a starting point. Entered through the self data structure.
139+
140+
Returns
141+
-------
142+
theta : array-like of shape = number_of_model_elements
143+
The estimated parameters of the model.
144+
145+
References
146+
----------
147+
- Wikipedia entry on ridge regression
148+
https://en.wikipedia.org/wiki/Ridge_regression
149+
150+
ridge_parm multiplied by the identity matrix (np.eye) favors models (theta) that
151+
have small size using an L2 norm. This prevents over fitting of the model.
152+
For applications where preventing overfitting is important, see, for example,
153+
D. J. Gauthier, E. Bollt, A. Griffith, W. A. S. Barbosa, ‘Next generation
154+
reservoir computing,’ Nat. Commun. 12, 5564 (2021).
155+
https://www.nature.com/articles/s41467-021-25801-2
156+
157+
"""
158+
self._check_linear_dependence_rows(psi)
159+
160+
y = y[self.max_lag :, 0].reshape(-1, 1)
161+
theta = (np.linalg.pinv(psi.T @ psi + self.ridge_param * np.eye(psi.shape[1])) @ psi.T @ y)
162+
return theta
119163

120164
def _unbiased_estimator(self, psi, y, theta, elag, max_lag, estimator):
121165
"""Estimate the model parameters using Extended Least Squares method.

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /