From e7de925373f19f9df5d18b1fd6a467e67533d304 Mon Sep 17 00:00:00 2001 From: lluni Date: Sat, 28 May 2022 03:14:52 +0200 Subject: [PATCH 1/3] Added support for creating matrices with random values --- .../java/de/lluni/javann/util/Utilities.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/main/java/de/lluni/javann/util/Utilities.java b/src/main/java/de/lluni/javann/util/Utilities.java index cadd566..bd98b08 100644 --- a/src/main/java/de/lluni/javann/util/Utilities.java +++ b/src/main/java/de/lluni/javann/util/Utilities.java @@ -14,6 +14,9 @@ import java.util.Random; public class Utilities { private static final double STANDARD_GAUSSIAN_FACTOR = 1.0d; + private static final double STANDARD_RANDOM_ORIGIN = -1.0d; + private static final double STANDARD_RANDOM_BOUND = 1.0d; + /** * Creates a matrix filled with ones * @param rows amount of rows @@ -50,6 +53,24 @@ public class Utilities { return gaussianMatrix(rows, columns, mean, stddev, STANDARD_GAUSSIAN_FACTOR); } + /** + * Creates a matrix with random values + * @param rows amount of rows + * @param columns amount of columns + * @param origin minimum random value + * @param bound maximum random value + * @return matrix with random values + */ + public static SimpleMatrix randomMatrix(int rows, int columns, double origin, double bound) { + Random random = new Random(); + return new SimpleMatrix(rows, columns, true, + random.doubles((long) rows * columns, origin, bound).toArray()); + } + + public static SimpleMatrix randomMatrix(int rows, int columns) { + return randomMatrix(rows, columns, STANDARD_RANDOM_ORIGIN, STANDARD_RANDOM_BOUND); + } + /** * Creates an array of evenly spaced values from the interval [start, end) * @param start start value From faa547564c7daf3a9cd3f5e8a37fcfe9fdd6136d Mon Sep 17 00:00:00 2001 From: lluni Date: Sat, 28 May 2022 03:19:49 +0200 Subject: [PATCH 2/3] Added support for choosing weight and bias initializers --- .../java/de/lluni/javann/Initializer.java | 8 +++++++ .../de/lluni/javann/examples/ExampleSine.java | 7 ++++--- .../de/lluni/javann/examples/ExampleXOR.java | 5 +++-- .../examples/ExampleXORAddedNeurons.java | 5 +++-- .../java/de/lluni/javann/layers/FCLayer.java | 21 ++++++++++++++++--- 5 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 src/main/java/de/lluni/javann/Initializer.java diff --git a/src/main/java/de/lluni/javann/Initializer.java b/src/main/java/de/lluni/javann/Initializer.java new file mode 100644 index 0000000..2458500 --- /dev/null +++ b/src/main/java/de/lluni/javann/Initializer.java @@ -0,0 +1,8 @@ +package de.lluni.javann; + +public enum Initializer { + ZEROS, + ONES, + GAUSSIAN, + RANDOM +} diff --git a/src/main/java/de/lluni/javann/examples/ExampleSine.java b/src/main/java/de/lluni/javann/examples/ExampleSine.java index 3dbddfb..c0701d2 100644 --- a/src/main/java/de/lluni/javann/examples/ExampleSine.java +++ b/src/main/java/de/lluni/javann/examples/ExampleSine.java @@ -1,5 +1,6 @@ package de.lluni.javann.examples; +import de.lluni.javann.Initializer; import de.lluni.javann.Network; import de.lluni.javann.functions.ActivationFunctions; import de.lluni.javann.functions.LossFunctions; @@ -46,11 +47,11 @@ public class ExampleSine { // create network and add layers Network network = new Network(); - network.addLayer(new FCLayer(8)); + network.addLayer(new FCLayer(8, Initializer.GAUSSIAN, Initializer.ONES)); network.addLayer(new ActivationLayer(ActivationFunctions::LeakyReLu, ActivationFunctions::LeakyReLuPrime)); - network.addLayer(new FCLayer(8)); + network.addLayer(new FCLayer(8, Initializer.GAUSSIAN, Initializer.ONES)); network.addLayer(new ActivationLayer(ActivationFunctions::LeakyReLu, ActivationFunctions::LeakyReLuPrime)); - network.addLayer(new FCLayer(1)); + network.addLayer(new FCLayer(1, Initializer.GAUSSIAN, Initializer.ONES)); // configure loss function for the network network.use(LossFunctions::MSE, LossFunctions::MSEPrime); diff --git a/src/main/java/de/lluni/javann/examples/ExampleXOR.java b/src/main/java/de/lluni/javann/examples/ExampleXOR.java index 0e42079..f647673 100644 --- a/src/main/java/de/lluni/javann/examples/ExampleXOR.java +++ b/src/main/java/de/lluni/javann/examples/ExampleXOR.java @@ -1,5 +1,6 @@ package de.lluni.javann.examples; +import de.lluni.javann.Initializer; import de.lluni.javann.Network; import de.lluni.javann.functions.ActivationFunctions; import de.lluni.javann.functions.LossFunctions; @@ -19,9 +20,9 @@ public class ExampleXOR { new SimpleMatrix(new double[][]{{0}})}; Network network = new Network(); - network.addLayer(new FCLayer(3)); + network.addLayer(new FCLayer(3, Initializer.RANDOM, Initializer.RANDOM)); network.addLayer(new ActivationLayer(ActivationFunctions::tanh, ActivationFunctions::tanhPrime)); - network.addLayer(new FCLayer(1)); + network.addLayer(new FCLayer(1, Initializer.RANDOM, Initializer.RANDOM)); network.addLayer(new ActivationLayer(ActivationFunctions::tanh, ActivationFunctions::tanhPrime)); network.use(LossFunctions::MSE, LossFunctions::MSEPrime); diff --git a/src/main/java/de/lluni/javann/examples/ExampleXORAddedNeurons.java b/src/main/java/de/lluni/javann/examples/ExampleXORAddedNeurons.java index 20d1da2..b19ab58 100644 --- a/src/main/java/de/lluni/javann/examples/ExampleXORAddedNeurons.java +++ b/src/main/java/de/lluni/javann/examples/ExampleXORAddedNeurons.java @@ -1,5 +1,6 @@ package de.lluni.javann.examples; +import de.lluni.javann.Initializer; import de.lluni.javann.Network; import de.lluni.javann.functions.ActivationFunctions; import de.lluni.javann.functions.LossFunctions; @@ -19,9 +20,9 @@ public class ExampleXORAddedNeurons { new SimpleMatrix(new double[][]{{0}})}; Network network = new Network(); - network.addLayer(new FCLayer(1)); + network.addLayer(new FCLayer(1, Initializer.RANDOM, Initializer.RANDOM)); network.addLayer(new ActivationLayer(ActivationFunctions::tanh, ActivationFunctions::tanhPrime)); - network.addLayer(new FCLayer(1)); + network.addLayer(new FCLayer(1, Initializer.RANDOM, Initializer.RANDOM)); network.addLayer(new ActivationLayer(ActivationFunctions::tanh, ActivationFunctions::tanhPrime)); network.addNeuron(0, 2); diff --git a/src/main/java/de/lluni/javann/layers/FCLayer.java b/src/main/java/de/lluni/javann/layers/FCLayer.java index 2c78c5c..8d2617f 100644 --- a/src/main/java/de/lluni/javann/layers/FCLayer.java +++ b/src/main/java/de/lluni/javann/layers/FCLayer.java @@ -1,5 +1,6 @@ package de.lluni.javann.layers; +import de.lluni.javann.Initializer; import de.lluni.javann.util.Utilities; import org.ejml.simple.SimpleMatrix; @@ -9,6 +10,8 @@ import java.util.Random; * Fully connected layer with n Neurons */ public class FCLayer extends Layer { + private final Initializer weightInit; + private final Initializer biasInit; private SimpleMatrix weights; private SimpleMatrix biases; private int numNeurons; @@ -18,14 +21,26 @@ public class FCLayer extends Layer { * Creates a fully connected layer with numNeurons neurons * @param numNeurons amount of neurons in this layer */ - public FCLayer(int numNeurons) { + public FCLayer(int numNeurons, Initializer weightInit, Initializer biasInit) { this.numNeurons = numNeurons; + this.weightInit = weightInit; + this.biasInit = biasInit; isInitialized = false; } private void initialize(int inputSize) { - this.weights = Utilities.gaussianMatrix(inputSize, numNeurons, 0, 1, 0.1d); - this.biases = Utilities.ones(1, numNeurons); + switch (weightInit) { + case ZEROS -> this.weights = new SimpleMatrix(inputSize, numNeurons); + case ONES -> this.weights = Utilities.ones(inputSize, numNeurons); + case GAUSSIAN -> this.weights = Utilities.gaussianMatrix(inputSize, numNeurons, 0, 1, 0.1d); + case RANDOM -> this.weights = Utilities.randomMatrix(inputSize, numNeurons); + } + switch (biasInit) { + case ZEROS -> this.biases = new SimpleMatrix(1, numNeurons); + case ONES -> this.biases = Utilities.ones(1, numNeurons); + case GAUSSIAN -> this.biases = Utilities.gaussianMatrix(1, numNeurons, 0, 1, 0.1d); + case RANDOM -> this.biases = Utilities.randomMatrix(1, numNeurons); + } this.isInitialized = true; } From c7154817eefae4cb4ed94066280b6c9871aadb13 Mon Sep 17 00:00:00 2001 From: lluni Date: Sat, 28 May 2022 03:28:00 +0200 Subject: [PATCH 3/3] Added support for choosing if the step size should decrease for each subsequent epoch --- src/main/java/de/lluni/javann/Network.java | 9 +++++++-- src/main/java/de/lluni/javann/examples/ExampleSine.java | 2 +- src/main/java/de/lluni/javann/examples/ExampleXOR.java | 2 +- .../de/lluni/javann/examples/ExampleXORAddedNeurons.java | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/java/de/lluni/javann/Network.java b/src/main/java/de/lluni/javann/Network.java index f03555b..d62648d 100644 --- a/src/main/java/de/lluni/javann/Network.java +++ b/src/main/java/de/lluni/javann/Network.java @@ -79,8 +79,9 @@ public class Network { * @param y_train labels * @param epochs amount of training iterations * @param learningRate step size of gradient descent + * @param optimize if step size should decrease for each subsequent epoch */ - public void fit(SimpleMatrix[] X_train, SimpleMatrix[] y_train, int epochs, double learningRate) { + public void fit(SimpleMatrix[] X_train, SimpleMatrix[] y_train, int epochs, double learningRate, boolean optimize) { int samples = X_train.length; for (int i = 0; i < epochs; i++) { @@ -98,7 +99,11 @@ public class Network { // backward propagation SimpleMatrix error = lossPrime.apply(y_train[j], output); for (int k = layers.size() - 1; k >= 0; k--) { - error = layers.get(k).backwardPropagation(error, learningRate / (i+1)); + if (optimize) { + error = layers.get(k).backwardPropagation(error, learningRate / (i+1)); + } else { + error = layers.get(k).backwardPropagation(error, learningRate); + } } } // calculate average error on all samples diff --git a/src/main/java/de/lluni/javann/examples/ExampleSine.java b/src/main/java/de/lluni/javann/examples/ExampleSine.java index c0701d2..1572b0a 100644 --- a/src/main/java/de/lluni/javann/examples/ExampleSine.java +++ b/src/main/java/de/lluni/javann/examples/ExampleSine.java @@ -57,7 +57,7 @@ public class ExampleSine { network.use(LossFunctions::MSE, LossFunctions::MSEPrime); // train network on X_train and y_train - network.fit(X_train, y_train, 100, 0.05d); + network.fit(X_train, y_train, 100, 0.05d, true); // predict X_test and output results to console SimpleMatrix[] output = network.predict(X_test); diff --git a/src/main/java/de/lluni/javann/examples/ExampleXOR.java b/src/main/java/de/lluni/javann/examples/ExampleXOR.java index f647673..7f4d56f 100644 --- a/src/main/java/de/lluni/javann/examples/ExampleXOR.java +++ b/src/main/java/de/lluni/javann/examples/ExampleXOR.java @@ -26,7 +26,7 @@ public class ExampleXOR { network.addLayer(new ActivationLayer(ActivationFunctions::tanh, ActivationFunctions::tanhPrime)); network.use(LossFunctions::MSE, LossFunctions::MSEPrime); - network.fit(X_train, y_train, 1000, 0.1d); + network.fit(X_train, y_train, 1000, 0.1d, false); SimpleMatrix[] output = network.predict(X_train); for (SimpleMatrix entry : output) { diff --git a/src/main/java/de/lluni/javann/examples/ExampleXORAddedNeurons.java b/src/main/java/de/lluni/javann/examples/ExampleXORAddedNeurons.java index b19ab58..af8c31a 100644 --- a/src/main/java/de/lluni/javann/examples/ExampleXORAddedNeurons.java +++ b/src/main/java/de/lluni/javann/examples/ExampleXORAddedNeurons.java @@ -27,7 +27,7 @@ public class ExampleXORAddedNeurons { network.addNeuron(0, 2); network.use(LossFunctions::MSE, LossFunctions::MSEPrime); - network.fit(X_train, y_train, 1000, 0.1d); + network.fit(X_train, y_train, 1000, 0.1d, false); SimpleMatrix[] output = network.predict(X_train); for (SimpleMatrix entry : output) {