patrick/little_learner
1
0
Fork 0
Code Issues Pull Requests Packages Releases Activity

Implement plane prediction (#11)

Browse Source
This commit is contained in:
Patrick Stevens
2023-04-07 20:41:49 +01:00
committed by GitHub
parent 3c964bc132
commit 753722d7ca
7 changed files with 574 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
little_learner_app/Cargo.toml
View File

@@ -9,4 +9,3 @@ edition = "2021"
immutable-chunkmap = "1.0.5"
ordered-float = "3.6.0"
little_learner = { path = "../little_learner" }
arrayvec = "0.7.2"

279
little_learner_app/src/main.rs
View File

@@ -6,9 +6,9 @@ mod with_tensor;
use core::hash::Hash;
use std::ops::{Add, AddAssign, Div, Mul, Neg};
use little_learner::auto_diff::{of_scalar, of_slice, to_scalar, Differentiable};
use little_learner::auto_diff::{grad, Differentiable, RankedDifferentiable};
use little_learner::loss::{l2_loss_2, predict_quadratic};
use little_learner::loss::{l2_loss_2, predict_plane};
use little_learner::scalar::Scalar;
use little_learner::traits::{Exp, One, Zero};
use ordered_float::NotNan;
@@ -24,16 +24,16 @@ where
v
}
struct GradientDescentHyper<A, const RANK: usize> {
struct GradientDescentHyper<A> {
learning_rate: A,
iterations: u32,
}
fn gradient_descent_step<A, F, const RANK: usize>(
fn gradient_descent_step<A, F, const RANK: usize, const PARAM_NUM: usize>(
f: &F,
theta: Differentiable<A, RANK>,
params: &GradientDescentHyper<A, RANK>,
) -> Differentiable<A, RANK>
theta: [Differentiable<A>; PARAM_NUM],
params: &GradientDescentHyper<A>,
) -> [Differentiable<A>; PARAM_NUM]
where
A: Clone
+ Mul<Output = A>
@@ -46,17 +46,33 @@ where
+ One
+ Eq
+ Exp,
F: Fn(Differentiable<A, RANK>) -> Differentiable<A, RANK>,
F: Fn(&[Differentiable<A>; PARAM_NUM]) -> RankedDifferentiable<A, RANK>,
{
let delta = Differentiable::grad(f, &theta);
Differentiable::map2(&theta, &delta, &|theta, delta| {
(*theta).clone() - (Scalar::make((params.learning_rate).clone()) * (*delta).clone())
let delta = grad(f, &theta);
let mut i = 0;
theta.map(|theta| {
let delta = &delta[i];
i += 1;
// For speed, you might want to truncate_dual this.
let learning_rate = Scalar::make((params.learning_rate).clone());
Differentiable::map2(
&theta,
&delta.map(&mut |s| s * learning_rate.clone()),
&|theta, delta| (*theta).clone() - (*delta).clone(),
)
})
}
fn main() {
let xs = [-1.0, 0.0, 1.0, 2.0, 3.0];
let ys = [2.55, 2.1, 4.35, 10.2, 18.25];
let plane_xs = [
[1.0, 2.05],
[1.0, 3.0],
[2.0, 2.0],
[2.0, 3.91],
[3.0, 6.13],
[4.0, 8.09],
];
let plane_ys = [13.99, 15.99, 18.0, 22.4, 30.2, 37.94];
let hyper = GradientDescentHyper {
learning_rate: NotNan::new(0.001).expect("not nan"),
@@ -64,48 +80,63 @@ fn main() {
};
let iterated = {
let xs = xs.map(|x| NotNan::new(x).expect("not nan"));
let ys = ys.map(|x| NotNan::new(x).expect("not nan"));
let xs = plane_xs.map(|x| {
[
NotNan::new(x[0]).expect("not nan"),
NotNan::new(x[1]).expect("not nan"),
]
});
let ys = plane_ys.map(|x| NotNan::new(x).expect("not nan"));
iterate(
&|theta| {
gradient_descent_step(
&|x| {
Differentiable::of_vector(vec![of_scalar(l2_loss_2(
predict_quadratic,
of_slice(&xs),
of_slice(&ys),
x,
))])
RankedDifferentiable::of_vector(vec![RankedDifferentiable::of_scalar(
l2_loss_2(
predict_plane,
RankedDifferentiable::of_slice_2::<_, 2>(&xs),
RankedDifferentiable::of_slice(ys),
x,
),
)])
},
theta,
&hyper,
)
},
of_slice(&[
NotNan::<f64>::zero(),
NotNan::<f64>::zero(),
NotNan::<f64>::zero(),
]),
[
RankedDifferentiable::of_slice([NotNan::zero(), NotNan::zero()]).to_unranked(),
Differentiable::Scalar(Scalar::zero()),
],
hyper.iterations,
)
};
println!(
"After iteration: {:?}",
Differentiable::to_vector(iterated)
let [theta0, theta1] = iterated;
let theta0 = theta0.attach_rank::<1>().expect("rank 1 tensor");
let theta1 = theta1.attach_rank::<0>().expect("rank 0 tensor");
assert_eq!(
theta0
.to_vector()
.into_iter()
.map(|x| to_scalar(x).real_part().into_inner())
.collect::<Vec<_>>()
.map(|x| x.to_scalar().real_part().into_inner())
.collect::<Vec<_>>(),
[3.97757644609063, 2.0496557321494446]
);
assert_eq!(
theta1.to_scalar().real_part().into_inner(),
5.786758464448078
);
}
#[cfg(test)]
mod tests {
use super::*;
use arrayvec::ArrayVec;
use little_learner::{
auto_diff::to_scalar,
loss::{predict_line_2, square},
auto_diff::grad,
loss::{l2_loss_2, predict_line_2, predict_line_2_unranked, predict_quadratic_unranked},
};
use crate::with_tensor::{l2_loss, predict_line};
@@ -116,9 +147,12 @@ mod tests {
let ys = [1.8, 1.2, 4.2, 3.3];
let loss = l2_loss_2(
predict_line_2,
of_slice(&xs),
of_slice(&ys),
of_slice(&[0.0, 0.0]),
RankedDifferentiable::of_slice(&xs),
RankedDifferentiable::of_slice(&ys),
&[
RankedDifferentiable::of_scalar(Scalar::zero()),
RankedDifferentiable::of_scalar(Scalar::zero()),
],
);
assert_eq!(*loss.real_part(), 33.21);
@@ -134,29 +168,39 @@ mod tests {
#[test]
fn grad_example() {
let input_vec = of_slice(&[NotNan::new(27.0).expect("not nan")]);
let input_vec = [Differentiable::Scalar(Scalar::make(
NotNan::new(27.0).expect("not nan"),
))];
let grad: Vec<_> = Differentiable::to_vector(Differentiable::grad(
|x| Differentiable::map(x, &mut |x| square(&x)),
let grad: Vec<_> = grad(
|x| {
RankedDifferentiable::of_scalar(
x[0].borrow_scalar().clone() * x[0].borrow_scalar().clone(),
)
},
&input_vec,
))
)
.into_iter()
.map(|x| to_scalar(x).real_part().into_inner())
.map(|x| x.into_scalar().real_part().into_inner())
.collect();
assert_eq!(grad, [54.0]);
}
#[test]
fn loss_gradient() {
let input_vec = of_slice(&[NotNan::<f64>::zero(), NotNan::<f64>::zero()]);
let zero = Scalar::<NotNan<f64>>::zero();
let input_vec = [
RankedDifferentiable::of_scalar(zero.clone()).to_unranked(),
RankedDifferentiable::of_scalar(zero).to_unranked(),
];
let xs = [2.0, 1.0, 4.0, 3.0].map(|x| NotNan::new(x).expect("not nan"));
let ys = [1.8, 1.2, 4.2, 3.3].map(|x| NotNan::new(x).expect("not nan"));
let grad = Differentiable::grad(
let grad = grad(
|x| {
Differentiable::of_vector(vec![of_scalar(l2_loss_2(
predict_line_2,
of_slice(&xs),
of_slice(&ys),
RankedDifferentiable::of_vector(vec![RankedDifferentiable::of_scalar(l2_loss_2(
predict_line_2_unranked,
RankedDifferentiable::of_slice(&xs),
RankedDifferentiable::of_slice(&ys),
x,
))])
},
@@ -164,9 +208,8 @@ mod tests {
);
assert_eq!(
Differentiable::to_vector(grad)
.into_iter()
.map(|x| *(to_scalar(x).real_part()))
grad.into_iter()
.map(|x| *(x.into_scalar().real_part()))
.collect::<Vec<_>>(),
[-63.0, -21.0]
);
@@ -174,13 +217,7 @@ mod tests {
#[test]
fn test_iterate() {
let f = |t: [i32; 3]| {
let mut vec = ArrayVec::<i32, 3>::new();
for i in t {
vec.push(i - 3);
}
vec.into_inner().unwrap()
};
let f = |t: [i32; 3]| t.map(|i| i - 3);
assert_eq!(iterate(&f, [1, 2, 3], 5u32), [-14, -13, -12]);
}
@@ -189,6 +226,8 @@ mod tests {
let xs = [2.0, 1.0, 4.0, 3.0];
let ys = [1.8, 1.2, 4.2, 3.3];
let zero = Scalar::<NotNan<f64>>::zero();
let hyper = GradientDescentHyper {
learning_rate: NotNan::new(0.01).expect("not nan"),
iterations: 1000,
@@ -200,24 +239,29 @@ mod tests {
&|theta| {
gradient_descent_step(
&|x| {
Differentiable::of_vector(vec![of_scalar(l2_loss_2(
predict_line_2,
of_slice(&xs),
of_slice(&ys),
x,
))])
RankedDifferentiable::of_vector(vec![RankedDifferentiable::of_scalar(
l2_loss_2(
predict_line_2_unranked,
RankedDifferentiable::of_slice(&xs),
RankedDifferentiable::of_slice(&ys),
x,
),
)])
},
theta,
&hyper,
)
},
of_slice(&[NotNan::<f64>::zero(), NotNan::<f64>::zero()]),
[
RankedDifferentiable::of_scalar(zero.clone()).to_unranked(),
RankedDifferentiable::of_scalar(zero).to_unranked(),
],
hyper.iterations,
)
};
let iterated = Differentiable::to_vector(iterated)
let iterated = iterated
.into_iter()
.map(|x| to_scalar(x).real_part().into_inner())
.map(|x| x.into_scalar().real_part().into_inner())
.collect::<Vec<_>>();
assert_eq!(iterated, vec![1.0499993623489503, 0.0000018747718457656533]);
@@ -228,6 +272,8 @@ mod tests {
let xs = [-1.0, 0.0, 1.0, 2.0, 3.0];
let ys = [2.55, 2.1, 4.35, 10.2, 18.25];
let zero = Scalar::<NotNan<f64>>::zero();
let hyper = GradientDescentHyper {
learning_rate: NotNan::new(0.001).expect("not nan"),
iterations: 1000,
@@ -240,35 +286,104 @@ mod tests {
&|theta| {
gradient_descent_step(
&|x| {
Differentiable::of_vector(vec![of_scalar(l2_loss_2(
predict_quadratic,
of_slice(&xs),
of_slice(&ys),
x,
))])
RankedDifferentiable::of_vector(vec![RankedDifferentiable::of_scalar(
l2_loss_2(
predict_quadratic_unranked,
RankedDifferentiable::of_slice(&xs),
RankedDifferentiable::of_slice(&ys),
x,
),
)])
},
theta,
&hyper,
)
},
of_slice(&[
NotNan::<f64>::zero(),
NotNan::<f64>::zero(),
NotNan::<f64>::zero(),
]),
[
RankedDifferentiable::of_scalar(zero.clone()).to_unranked(),
RankedDifferentiable::of_scalar(zero.clone()).to_unranked(),
RankedDifferentiable::of_scalar(zero).to_unranked(),
],
hyper.iterations,
)
};
let iterated = Differentiable::to_vector(iterated)
let iterated = iterated
.into_iter()
.map(|x| to_scalar(x).real_part().into_inner())
.map(|x| x.into_scalar().real_part().into_inner())
.collect::<Vec<_>>();
println!("{:?}", iterated);
assert_eq!(
iterated,
[2.0546423148479684, 0.9928606519360353, 1.4787394427094362]
);
}
#[test]
fn optimise_plane() {
let plane_xs = [
[1.0, 2.05],
[1.0, 3.0],
[2.0, 2.0],
[2.0, 3.91],
[3.0, 6.13],
[4.0, 8.09],
];
let plane_ys = [13.99, 15.99, 18.0, 22.4, 30.2, 37.94];
let hyper = GradientDescentHyper {
learning_rate: NotNan::new(0.001).expect("not nan"),
iterations: 1000,
};
let iterated = {
let xs = plane_xs.map(|x| {
[
NotNan::new(x[0]).expect("not nan"),
NotNan::new(x[1]).expect("not nan"),
]
});
let ys = plane_ys.map(|x| NotNan::new(x).expect("not nan"));
iterate(
&|theta| {
gradient_descent_step(
&|x| {
RankedDifferentiable::of_vector(vec![RankedDifferentiable::of_scalar(
l2_loss_2(
predict_plane,
RankedDifferentiable::of_slice_2::<_, 2>(&xs),
RankedDifferentiable::of_slice(ys),
x,
),
)])
},
theta,
&hyper,
)
},
[
RankedDifferentiable::of_slice([NotNan::zero(), NotNan::zero()]).to_unranked(),
Differentiable::Scalar(Scalar::zero()),
],
hyper.iterations,
)
};
let [theta0, theta1] = iterated;
let theta0 = theta0.attach_rank::<1>().expect("rank 1 tensor");
let theta1 = theta1.attach_rank::<0>().expect("rank 0 tensor");
assert_eq!(
theta0
.to_vector()
.into_iter()
.map(|x| x.to_scalar().real_part().into_inner())
.collect::<Vec<_>>(),
[3.97757644609063, 2.0496557321494446]
);
assert_eq!(
theta1.to_scalar().real_part().into_inner(),
5.786758464448078
);
}
}