diff --git a/examples/example_sine.rs b/examples/example_sine.rs index 45da5b8..206a0ff 100644 --- a/examples/example_sine.rs +++ b/examples/example_sine.rs @@ -3,20 +3,24 @@ extern crate rust_nn; use std::error::Error; use std::f64::consts::PI; -use ndarray_rand::RandomExt; +use ndarray::Array1; use ndarray_rand::rand_distr::Uniform; +use ndarray_rand::RandomExt; use plotters::prelude::*; -use rust_nn::Network; use rust_nn::functions::{activation_functions, loss_functions}; use rust_nn::layers::activation_layer::ActivationLayer; use rust_nn::layers::fc_layer::{FCLayer, Initializer}; -use ndarray::Array1; +use rust_nn::Network; fn main() -> Result<(), Box> { // training data let training_interval = (0.0f64, 2.0f64 * PI); let steps = 100000; - let training_values = Array1::random(steps, Uniform::new(training_interval.0, training_interval.1)).to_vec(); + let training_values = Array1::random( + steps, + Uniform::new(training_interval.0, training_interval.1), + ) + .to_vec(); let mut x_train = Vec::new(); let mut y_train = Vec::new(); for x in training_values { @@ -42,19 +46,23 @@ fn main() -> Result<(), Box> { network.add_layer(Box::new(FCLayer::new( 8, Initializer::GaussianWFactor(0.0, 1.0, 0.1), - Initializer::GaussianWFactor(0.0, 1.0, 0.1) + Initializer::GaussianWFactor(0.0, 1.0, 0.1), + ))); + network.add_layer(Box::new(ActivationLayer::new( + activation_functions::Type::LeakyRelu, ))); - network.add_layer(Box::new(ActivationLayer::new(activation_functions::Type::LeakyRelu))); network.add_layer(Box::new(FCLayer::new( 8, Initializer::GaussianWFactor(0.0, 1.0, 0.1), - Initializer::GaussianWFactor(0.0, 1.0, 0.1) + Initializer::GaussianWFactor(0.0, 1.0, 0.1), + ))); + network.add_layer(Box::new(ActivationLayer::new( + activation_functions::Type::LeakyRelu, ))); - network.add_layer(Box::new(ActivationLayer::new(activation_functions::Type::LeakyRelu))); network.add_layer(Box::new(FCLayer::new( 1, Initializer::GaussianWFactor(0.0, 1.0, 0.1), - Initializer::GaussianWFactor(0.0, 1.0, 0.1) + Initializer::GaussianWFactor(0.0, 1.0, 0.1), ))); // train network on training data @@ -79,20 +87,26 @@ fn main() -> Result<(), Box> { .draw()?; // add the first plot - let data1: Vec<(f64,f64)> = x_test.iter().zip(y_test_true.iter()) + let data1: Vec<(f64, f64)> = x_test + .iter() + .zip(y_test_true.iter()) .map(|(x, y)| (x[0], y[0])) .collect(); chart - .draw_series(LineSeries::new(data1, &RED)).unwrap() + .draw_series(LineSeries::new(data1, &RED)) + .unwrap() .label("true values") .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 1, y)], &RED)); // add the second plot - let data2: Vec<(f64,f64)> = x_test.iter().zip(y_test_pred.iter()) + let data2: Vec<(f64, f64)> = x_test + .iter() + .zip(y_test_pred.iter()) .map(|(x, y)| (x[0], y[0])) .collect(); chart - .draw_series(LineSeries::new(data2, &BLUE)).unwrap() + .draw_series(LineSeries::new(data2, &BLUE)) + .unwrap() .label("predicted values") .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 1, y)], &BLUE)); diff --git a/examples/example_xor.rs b/examples/example_xor.rs index 6d2346b..9934aff 100644 --- a/examples/example_xor.rs +++ b/examples/example_xor.rs @@ -1,10 +1,10 @@ extern crate rust_nn; -use rust_nn::Network; +use ndarray::array; use rust_nn::functions::{activation_functions, loss_functions}; use rust_nn::layers::activation_layer::ActivationLayer; use rust_nn::layers::fc_layer::{FCLayer, Initializer}; -use ndarray::array; +use rust_nn::Network; fn main() { // training data @@ -12,20 +12,15 @@ fn main() { array![0.0, 0.0], array![0.0, 1.0], array![1.0, 0.0], - array![1.0, 1.0] - ]; - let y_train = vec![ - array![0.0], - array![1.0], - array![1.0], - array![0.0] + array![1.0, 1.0], ]; + let y_train = vec![array![0.0], array![1.0], array![1.0], array![0.0]]; // test data - let x_test= vec![ + let x_test = vec![ array![0.0, 0.0], array![0.0, 1.0], array![1.0, 0.0], - array![1.0, 1.0] + array![1.0, 1.0], ]; // initialize neural network @@ -35,15 +30,19 @@ fn main() { network.add_layer(Box::new(FCLayer::new( 3, Initializer::Gaussian(0.0, 1.0), - Initializer::Gaussian(0.0, 1.0) + Initializer::Gaussian(0.0, 1.0), + ))); + network.add_layer(Box::new(ActivationLayer::new( + activation_functions::Type::Tanh, ))); - network.add_layer(Box::new(ActivationLayer::new(activation_functions::Type::Tanh))); network.add_layer(Box::new(FCLayer::new( 1, Initializer::Gaussian(0.0, 1.0), - Initializer::Gaussian(0.0, 1.0) + Initializer::Gaussian(0.0, 1.0), + ))); + network.add_layer(Box::new(ActivationLayer::new( + activation_functions::Type::Tanh, ))); - network.add_layer(Box::new(ActivationLayer::new(activation_functions::Type::Tanh))); // train network on training data network.fit(x_train, y_train, 1000, 0.1, false); @@ -58,4 +57,4 @@ fn main() { prediction.map_mut(|x| *x = x.round()); print!("prediction: {}\n", prediction); } -} \ No newline at end of file +} diff --git a/src/functions/activation_functions.rs b/src/functions/activation_functions.rs index ac3b6e4..bafd96c 100644 --- a/src/functions/activation_functions.rs +++ b/src/functions/activation_functions.rs @@ -6,16 +6,21 @@ pub enum Type { Logistic, Tanh, Relu, - LeakyRelu + LeakyRelu, } -pub fn parse_type(t: Type) -> (fn(&Array1) -> Array1, fn(&Array1) -> Array1) { +pub fn parse_type( + t: Type, +) -> ( + fn(&Array1) -> Array1, + fn(&Array1) -> Array1, +) { match t { Type::Identity => (identity, identity_prime), Type::Logistic => (logistic, logistic_prime), Type::Tanh => (tanh, tanh_prime), Type::Relu => (relu, relu_prime), - Type::LeakyRelu => (leaky_relu, leaky_relu_prime) + Type::LeakyRelu => (leaky_relu, leaky_relu_prime), } } @@ -78,7 +83,7 @@ pub fn relu(matrix: &Array1) -> Array1 { pub fn relu_prime(matrix: &Array1) -> Array1 { let mut result = matrix.clone(); for x in result.iter_mut() { - *x = if (*x) <= 0.0 {0.0} else {1.0}; + *x = if (*x) <= 0.0 { 0.0 } else { 1.0 }; } result } @@ -94,7 +99,7 @@ pub fn leaky_relu(matrix: &Array1) -> Array1 { pub fn leaky_relu_prime(matrix: &Array1) -> Array1 { let mut result = matrix.clone(); for x in result.iter_mut() { - *x = if (*x) <= 0.0 {0.001} else {1.0}; + *x = if (*x) <= 0.0 { 0.001 } else { 1.0 }; } result } diff --git a/src/functions/loss_functions.rs b/src/functions/loss_functions.rs index 5f15e6e..4eda8fe 100644 --- a/src/functions/loss_functions.rs +++ b/src/functions/loss_functions.rs @@ -2,13 +2,18 @@ use ndarray::{Array1, ArrayView1}; pub enum Type { MSE, - MAE + MAE, } -pub fn parse_type(t: Type) -> (fn(ArrayView1, ArrayView1) -> f64, fn(ArrayView1, ArrayView1) -> Array1) { +pub fn parse_type( + t: Type, +) -> ( + fn(ArrayView1, ArrayView1) -> f64, + fn(ArrayView1, ArrayView1) -> Array1, +) { match t { Type::MSE => (mse, mse_prime), - Type::MAE => (mae, mae_prime) + Type::MAE => (mae, mae_prime), } } diff --git a/src/layers/activation_layer.rs b/src/layers/activation_layer.rs index 6ba3c54..61324c4 100644 --- a/src/layers/activation_layer.rs +++ b/src/layers/activation_layer.rs @@ -1,13 +1,13 @@ -use ndarray::{Array1, arr1, ArrayView1}; +use ndarray::{arr1, Array1, ArrayView1}; -use crate::functions::activation_functions::*; use super::Layer; +use crate::functions::activation_functions::*; pub struct ActivationLayer { input: Array1, output: Array1, activation: fn(&Array1) -> Array1, - activation_prime: fn(&Array1) -> Array1 + activation_prime: fn(&Array1) -> Array1, } impl ActivationLayer { @@ -17,7 +17,7 @@ impl ActivationLayer { input: arr1(&[]), output: arr1(&[]), activation, - activation_prime + activation_prime, } } } @@ -36,5 +36,4 @@ impl Layer for ActivationLayer { temp.zip_mut_with(&output_error, |x, y| *x *= y); temp } - } diff --git a/src/layers/fc_layer.rs b/src/layers/fc_layer.rs index cac8209..1aa0202 100644 --- a/src/layers/fc_layer.rs +++ b/src/layers/fc_layer.rs @@ -1,8 +1,8 @@ extern crate ndarray; -use ndarray::{Array1, Array2, arr1, arr2, Array, ArrayView1, ShapeBuilder}; -use ndarray_rand::RandomExt; +use ndarray::{arr1, arr2, Array, Array1, Array2, ArrayView1, ShapeBuilder}; use ndarray_rand::rand_distr::{Normal, Uniform}; +use ndarray_rand::RandomExt; use super::Layer; @@ -11,21 +11,25 @@ pub enum Initializer { Ones, Gaussian(f64, f64), GaussianWFactor(f64, f64, f64), - Uniform(f64, f64) + Uniform(f64, f64), } impl Initializer { pub fn init(&self, shape: Sh) -> Array where - Sh: ShapeBuilder, D: ndarray::Dimension + Sh: ShapeBuilder, + D: ndarray::Dimension, { match self { Self::Zeros => Array::zeros(shape), Self::Ones => Array::ones(shape), - Self::Gaussian(mean, stddev) => Array::random(shape, Normal::new(*mean, *stddev).unwrap()), - Self::GaussianWFactor(mean, stddev, factor) - => Array::random(shape, Normal::new(*mean, *stddev).unwrap()) * *factor, - Self::Uniform(low, high) => Array::random(shape, Uniform::new(low, high)) + Self::Gaussian(mean, stddev) => { + Array::random(shape, Normal::new(*mean, *stddev).unwrap()) + } + Self::GaussianWFactor(mean, stddev, factor) => { + Array::random(shape, Normal::new(*mean, *stddev).unwrap()) * *factor + } + Self::Uniform(low, high) => Array::random(shape, Uniform::new(low, high)), } } } @@ -42,7 +46,11 @@ pub struct FCLayer { } impl FCLayer { - pub fn new(num_neurons: usize, weight_initializer: Initializer, bias_initializer: Initializer) -> Self { + pub fn new( + num_neurons: usize, + weight_initializer: Initializer, + bias_initializer: Initializer, + ) -> Self { FCLayer { num_neurons, is_initialized: false, @@ -51,7 +59,7 @@ impl FCLayer { input: arr1(&[]), output: arr1(&[]), weights: arr2(&[[]]), - biases: arr1(&[]) + biases: arr1(&[]), } } @@ -75,11 +83,18 @@ impl Layer for FCLayer { fn backward_pass(&mut self, output_error: ArrayView1, learning_rate: f64) -> Array1 { let input_error = output_error.dot(&self.weights.t()); - let delta_weights = - self.input.to_owned().into_shape((self.input.len(), 1usize)).unwrap() - .dot(&output_error.into_shape((1usize, output_error.len())).unwrap()); + let delta_weights = self + .input + .to_owned() + .into_shape((self.input.len(), 1usize)) + .unwrap() + .dot( + &output_error + .into_shape((1usize, output_error.len())) + .unwrap(), + ); self.weights = &self.weights + learning_rate * &delta_weights; self.biases = &self.biases + learning_rate * &output_error; input_error } -} \ No newline at end of file +} diff --git a/src/lib.rs b/src/lib.rs index eef17fc..f757376 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,7 @@ use ndarray::{Array1, ArrayView1}; pub struct Network { layers: Vec>, loss: fn(ArrayView1, ArrayView1) -> f64, - loss_prime: fn(ArrayView1, ArrayView1) -> Array1 + loss_prime: fn(ArrayView1, ArrayView1) -> Array1, } impl Network { @@ -17,7 +17,7 @@ impl Network { Network { layers: vec![], loss, - loss_prime + loss_prime, } } @@ -41,7 +41,14 @@ impl Network { result } - pub fn fit(&mut self, x_train: Vec>, y_train: Vec>, epochs: usize, learning_rate: f64, trivial_optimize: bool) { + pub fn fit( + &mut self, + x_train: Vec>, + y_train: Vec>, + epochs: usize, + learning_rate: f64, + trivial_optimize: bool, + ) { assert!(x_train.len() > 0); assert!(x_train.len() == y_train.len()); let num_samples = x_train.len(); @@ -63,7 +70,7 @@ impl Network { let mut error = (self.loss_prime)(y_train[j].view(), output.view()); for layer in self.layers.iter_mut().rev() { if trivial_optimize { - error = layer.backward_pass(error.view(), learning_rate / (i+1) as f64); + error = layer.backward_pass(error.view(), learning_rate / (i + 1) as f64); } else { error = layer.backward_pass(error.view(), learning_rate); } @@ -71,7 +78,7 @@ impl Network { } // calculate average error on all samples err /= num_samples as f64; - println!("epoch {}/{} error={}", i+1, epochs, err); + println!("epoch {}/{} error={}", i + 1, epochs, err); } } }