From cf24728f353ddd47b6fc56fb7f16ea9b359abe2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20Fenyvesi?= Date: Wed, 6 May 2020 14:56:22 +0200 Subject: [PATCH] tests for auto-attach and manual attach; improve JavaDoc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tamás Fenyvesi --- .../arbiter/optimize/TestBasic.java | 307 +++++++++++------- .../ui/TestVertxUIMultiSession.java | 4 +- 2 files changed, 195 insertions(+), 116 deletions(-) diff --git a/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/TestBasic.java b/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/TestBasic.java index 025ce85c6..08f130369 100644 --- a/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/TestBasic.java +++ b/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/TestBasic.java @@ -16,6 +16,7 @@ package org.deeplearning4j.arbiter.optimize; +import io.netty.handler.codec.http.HttpResponseStatus; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.BaseDL4JTest; @@ -65,12 +66,17 @@ import org.nd4j.linalg.function.Function; import org.nd4j.linalg.lossfunctions.LossFunctions; import java.io.File; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.util.*; import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Created by Alex on 19/07/2017. */ @@ -88,112 +94,15 @@ public class TestBasic extends BaseDL4JTest { UIServer.getInstance(); - Thread.sleep(1000000); + Thread.sleep(1000_000); } - @Test - @Ignore - public void testBasicUiMultiSession() throws Exception { - - Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); - - MultiLayerSpace mls = new MultiLayerSpace.Builder() - .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) - .l2(new ContinuousParameterSpace(0.0001, 0.05)) - .addLayer( - new ConvolutionLayerSpace.Builder().nIn(1) - .nOut(new IntegerParameterSpace(5, 30)) - .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3}, - new int[]{4, 4}, new int[]{5, 5})) - .stride(new DiscreteParameterSpace<>(new int[]{1, 1}, - new int[]{2, 2})) - .activation(new DiscreteParameterSpace<>(Activation.RELU, - Activation.SOFTPLUS, Activation.LEAKYRELU)) - .build()) - .addLayer(new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 128)) - .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) - .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers - .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) - .lossFunction(LossFunctions.LossFunction.MCXENT).build()) - .setInputType(InputType.convolutionalFlat(28, 28, 1)) - .build(); - Map commands = new HashMap<>(); -// commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); - - //Define configuration: - CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); - DataProvider dataProvider = new MnistDataSetProvider(); - - - String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnist\\").getAbsolutePath(); - - File f = new File(modelSavePath); - if (f.exists()) - f.delete(); - f.mkdir(); - if (!f.exists()) - throw new RuntimeException(); - - OptimizationConfiguration configuration = - new OptimizationConfiguration.Builder() - .candidateGenerator(candidateGenerator).dataProvider(dataProvider) - .modelSaver(new FileModelSaver(modelSavePath)) - .scoreFunction(new TestSetLossScoreFunction(true)) - .terminationConditions(new MaxTimeCondition(120, TimeUnit.MINUTES), - new MaxCandidatesCondition(100)) - .build(); - - IOptimizationRunner runner = - new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); - - // add 3 different sessions to the same execution - HashMap statsStorageForSession = new HashMap<>(); - for (int i = 0; i < 3; i++) { - StatsStorage ss = new InMemoryStatsStorage(); - @NonNull String sessionId = "sid" + i; - statsStorageForSession.put(sessionId, ss); - StatusListener sl = new ArbiterStatusListener(sessionId, ss); - runner.addListeners(sl); - } - - Function statsStorageProvider = statsStorageForSession::get; - UIServer uIServer = UIServer.getInstance(true, statsStorageProvider); - String serverAddress = uIServer.getAddress(); - for (String sessionId : statsStorageForSession.keySet()) { - log.info("Arbiter session can be attached at {}/arbiter/{}", serverAddress, sessionId); - } - - runner.execute(); - - Thread.sleep(1000000); - } - - @Test @Ignore public void testBasicMnist() throws Exception { Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); - MultiLayerSpace mls = new MultiLayerSpace.Builder() - .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) - .l2(new ContinuousParameterSpace(0.0001, 0.05)) - .addLayer( - new ConvolutionLayerSpace.Builder().nIn(1) - .nOut(new IntegerParameterSpace(5, 30)) - .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3}, - new int[]{4, 4}, new int[]{5, 5})) - .stride(new DiscreteParameterSpace<>(new int[]{1, 1}, - new int[]{2, 2})) - .activation(new DiscreteParameterSpace<>(Activation.RELU, - Activation.SOFTPLUS, Activation.LEAKYRELU)) - .build()) - .addLayer(new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 128)) - .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) - .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers - .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) - .lossFunction(LossFunctions.LossFunction.MCXENT).build()) - .setInputType(InputType.convolutionalFlat(28, 28, 1)) - .build(); + MultiLayerSpace mls = getMultiLayerSpaceMnist(); Map commands = new HashMap<>(); // commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); @@ -230,7 +139,30 @@ public class TestBasic extends BaseDL4JTest { UIServer.getInstance().attach(ss); runner.execute(); - Thread.sleep(100000); + Thread.sleep(1000_000); + } + + private static MultiLayerSpace getMultiLayerSpaceMnist() { + return new MultiLayerSpace.Builder() + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.05)) + .addLayer( + new ConvolutionLayerSpace.Builder().nIn(1) + .nOut(new IntegerParameterSpace(5, 30)) + .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3}, + new int[]{4, 4}, new int[]{5, 5})) + .stride(new DiscreteParameterSpace<>(new int[]{1, 1}, + new int[]{2, 2})) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.SOFTPLUS, Activation.LEAKYRELU)) + .build()) + .addLayer(new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 128)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers + .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)) + .build(); } @Test @@ -319,7 +251,7 @@ public class TestBasic extends BaseDL4JTest { .build(); //Define configuration: - CandidateGenerator candidateGenerator = new RandomSearchGenerator(cgs, Collections.EMPTY_MAP); + CandidateGenerator candidateGenerator = new RandomSearchGenerator(cgs); DataProvider dataProvider = new MnistDataSetProvider(); @@ -417,7 +349,7 @@ public class TestBasic extends BaseDL4JTest { UIServer.getInstance().attach(ss); runner.execute(); - Thread.sleep(100000); + Thread.sleep(1000_000); } @@ -482,7 +414,7 @@ public class TestBasic extends BaseDL4JTest { UIServer.getInstance().attach(ss); runner.execute(); - Thread.sleep(100000); + Thread.sleep(1000_000); } @@ -519,7 +451,7 @@ public class TestBasic extends BaseDL4JTest { .build(); //Define configuration: - CandidateGenerator candidateGenerator = new RandomSearchGenerator(cgs, Collections.EMPTY_MAP); + CandidateGenerator candidateGenerator = new RandomSearchGenerator(cgs); DataProvider dataProvider = new MnistDataSetProvider(); @@ -551,13 +483,17 @@ public class TestBasic extends BaseDL4JTest { UIServer.getInstance().attach(ss); runner.execute(); - Thread.sleep(100000); + Thread.sleep(1000_000); } + /** + * Visualize multiple optimization sessions run one after another on single-session mode UI + * @throws InterruptedException if current thread has been interrupted + */ @Test @Ignore - public void testBasicMnistMultipleSessions() throws Exception { + public void testBasicMnistMultipleSessions() throws InterruptedException { MultiLayerSpace mls = new MultiLayerSpace.Builder() .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) @@ -585,8 +521,10 @@ public class TestBasic extends BaseDL4JTest { //Define configuration: CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); - DataProvider dataProvider = new MnistDataSetProvider(); + Class ds = MnistDataSource.class; + Properties dsp = new Properties(); + dsp.setProperty("minibatch", "8"); String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnist\\").getAbsolutePath(); @@ -599,7 +537,7 @@ public class TestBasic extends BaseDL4JTest { OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() - .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .candidateGenerator(candidateGenerator).dataSource(ds, dsp) .modelSaver(new FileModelSaver(modelSavePath)) .scoreFunction(new TestSetLossScoreFunction(true)) .terminationConditions(new MaxTimeCondition(1, TimeUnit.MINUTES), @@ -621,7 +559,7 @@ public class TestBasic extends BaseDL4JTest { candidateGenerator = new RandomSearchGenerator(mls, commands); configuration = new OptimizationConfiguration.Builder() - .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .candidateGenerator(candidateGenerator).dataSource(ds, dsp) .modelSaver(new FileModelSaver(modelSavePath)) .scoreFunction(new TestSetLossScoreFunction(true)) .terminationConditions(new MaxTimeCondition(1, TimeUnit.MINUTES), @@ -636,7 +574,148 @@ public class TestBasic extends BaseDL4JTest { runner.execute(); - Thread.sleep(100000); + Thread.sleep(1000_000); + } + + /** + * Auto-attach multiple optimization sessions to multi-session mode UI + * @throws IOException if could not connect to the server + */ + @Test + public void testUiMultiSessionAutoAttach() throws IOException { + + //Define configuration: + MultiLayerSpace mls = getMultiLayerSpaceMnist(); + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls); + + Class ds = MnistDataSource.class; + Properties dsp = new Properties(); + dsp.setProperty("minibatch", "8"); + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestMultiSessionAutoAttach\\") + .getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataSource(ds, dsp) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(10, TimeUnit.SECONDS), + new MaxCandidatesCondition(1)) + .build(); + + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + + // add 3 different sessions to the same execution + HashMap statsStorageForSession = new HashMap<>(); + for (int i = 0; i < 3; i++) { + StatsStorage ss = new InMemoryStatsStorage(); + @NonNull String sessionId = "sid" + i; + statsStorageForSession.put(sessionId, ss); + StatusListener sl = new ArbiterStatusListener(sessionId, ss); + runner.addListeners(sl); + } + + Function statsStorageProvider = statsStorageForSession::get; + UIServer uIServer = UIServer.getInstance(true, statsStorageProvider); + String serverAddress = uIServer.getAddress(); + + runner.execute(); + + for (String sessionId : statsStorageForSession.keySet()) { + /* + * Visiting /arbiter/:sessionId to auto-attach StatsStorage + */ + String sessionUrl = sessionUrl(uIServer.getAddress(), sessionId); + HttpURLConnection conn = (HttpURLConnection) new URL(sessionUrl).openConnection(); + conn.connect(); + + log.info("Checking auto-attaching Arbiter session at {}", sessionUrl(serverAddress, sessionId)); + assertEquals(HttpResponseStatus.OK.code(), conn.getResponseCode()); + assertTrue(uIServer.isAttached(statsStorageForSession.get(sessionId))); + } + } + + /** + * Attach multiple optimization sessions to multi-session mode UI by manually visiting session URL + * @throws Exception if an error occurred + */ + @Test + @Ignore + public void testUiMultiSessionManualAttach() throws Exception { + Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); + + //Define configuration: + MultiLayerSpace mls = getMultiLayerSpaceMnist(); + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls); + + Class ds = MnistDataSource.class; + Properties dsp = new Properties(); + dsp.setProperty("minibatch", "8"); + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnist\\") + .getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataSource(ds, dsp) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(10, TimeUnit.MINUTES), + new MaxCandidatesCondition(10)) + .build(); + + + // parallel execution of multiple optimization sessions + HashMap statsStorageForSession = new HashMap<>(); + for (int i = 0; i < 3; i++) { + String sessionId = "sid" + i; + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + StatsStorage ss = new InMemoryStatsStorage(); + statsStorageForSession.put(sessionId, ss); + StatusListener sl = new ArbiterStatusListener(sessionId, ss); + runner.addListeners(sl); + // Asynchronous execution + new Thread(runner::execute).start(); + } + + Function statsStorageProvider = statsStorageForSession::get; + UIServer uIServer = UIServer.getInstance(true, statsStorageProvider); + String serverAddress = uIServer.getAddress(); + + for (String sessionId : statsStorageForSession.keySet()) { + log.info("Arbiter session can be attached at {}", sessionUrl(serverAddress, sessionId)); + } + + Thread.sleep(1000_000); + } + + + /** + * Get URL for arbiter session on given server address + * @param serverAddress server address, e.g.: http://localhost:9000 + * @param sessionId session ID (will be URL-encoded) + * @return URL + * @throws UnsupportedEncodingException if the character encoding is not supported + */ + private static String sessionUrl(String serverAddress, String sessionId) throws UnsupportedEncodingException { + return String.format("%s/arbiter/%s", serverAddress, URLEncoder.encode(sessionId, "UTF-8")); } private static class MnistDataSetProvider implements DataProvider { diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java b/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java index c9577d4a3..0f3f50d41 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java +++ b/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java @@ -197,9 +197,9 @@ public class TestVertxUIMultiSession extends BaseDL4JTest { } /** - * Get URL-encoded URL for training session on given server address + * Get URL for training session on given server address * @param serverAddress server address - * @param sessionId session ID + * @param sessionId session ID (will be URL-encoded) * @return URL * @throws UnsupportedEncodingException if the used encoding is not supported */