Slim down the e2e tests (#1685)

- Remove connection-oriented parameter cases for non-connection focused tests
  - For instance, a test about sending messages does not need to be run for both x509 authentication and SAS authentication. Nor does it need to be run for websocket vs TCP
  - Device/Module cases are still worth testing here because they use different service endpoints when publishing telemetry/methods
- Add connection test class for connection-oriented tests
  - Open/Close connection using all supported permutations of authentication, proxy/no proxy, transport protocol, device/module, TCP vs websocket
- Reorganize/cleanup some test code
  - Some "common" functions were only being used by one test method so they were moved there
- Rewrite some tests to use synchronous functions to simplify test code, remove helpers previously used to wait for async calls to finish

Author: timtay-microsoft

iot-e2e-tests/android/app/src/androidTest/java/com/microsoft/azure/sdk/iot/android/iothub/HubTierConnectionAndroidRunner.java

@@ -0,0 +1,19 @@
+package com.microsoft.azure.sdk.iot.android.iothub.connection;
+
+import com.microsoft.azure.sdk.iot.android.helper.TestGroup11;
+import com.microsoft.azure.sdk.iot.device.IotHubClientProtocol;
+import com.microsoft.azure.sdk.iot.service.auth.AuthenticationType;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import tests.integration.com.microsoft.azure.sdk.iot.helpers.ClientType;
+import tests.integration.com.microsoft.azure.sdk.iot.iothub.connection.ConnectionTests;
+
+@TestGroup11
+@RunWith(Parameterized.class)
+public class ConnectionTestsAndroidRunner extends ConnectionTests
+{
+    public ConnectionTestsAndroidRunner(IotHubClientProtocol protocol, AuthenticationType authenticationType, ClientType clientType, boolean withProxy) throws Exception
+    {
+        super(protocol, authenticationType, clientType, withProxy);
+    }
+}

iot-e2e-tests/android/app/src/androidTest/java/com/microsoft/azure/sdk/iot/android/iothub/connection/ConnectionTestsAndroidRunner.java

@@ -19,8 +19,8 @@
 @RunWith(Parameterized.class)
 public class SendMessagesErrInjAndroidRunner extends SendMessagesErrInjTests
 {
-    public SendMessagesErrInjAndroidRunner(IotHubClientProtocol protocol, AuthenticationType authenticationType, ClientType clientType, boolean useHttpProxy) throws Exception
+    public SendMessagesErrInjAndroidRunner(IotHubClientProtocol protocol, AuthenticationType authenticationType, ClientType clientType) throws Exception
     {
-        super(protocol, authenticationType, clientType, useHttpProxy);
+        super(protocol, authenticationType, clientType);
     }
 }

iot-e2e-tests/android/app/src/androidTest/java/com/microsoft/azure/sdk/iot/android/iothub/errorinjection/SendMessagesErrInjAndroidRunner.java

@@ -19,8 +19,8 @@
 @RunWith(Parameterized.class)
 public class SendMessagesAndroidRunner extends SendMessagesTests
 {
-    public SendMessagesAndroidRunner(IotHubClientProtocol protocol, AuthenticationType authenticationType, ClientType clientType, boolean useHttpProxy) throws Exception
+    public SendMessagesAndroidRunner(IotHubClientProtocol protocol, AuthenticationType authenticationType, ClientType clientType) throws Exception
     {
-        super(protocol, authenticationType, clientType, useHttpProxy);
+        super(protocol, authenticationType, clientType);
     }
 }

iot-e2e-tests/android/app/src/androidTest/java/com/microsoft/azure/sdk/iot/android/iothub/messaging/SendMessagesAndroidRunner.java

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-package tests.integration.com.microsoft.azure.sdk.iot.iothub.setup;
+package tests.integration.com.microsoft.azure.sdk.iot.helpers;
 
 import lombok.Getter;
 import lombok.Setter;

iot-e2e-tests/common/src/test/java/tests/integration/com/microsoft/azure/sdk/iot/iothub/setup/CustomObject.java iot-e2e-tests/common/src/test/java/tests/integration/com/microsoft/azure/sdk/iot/helpers/CustomObject.java

@@ -22,246 +22,9 @@
  */
 public class IotHubServicesCommon
 {
-    //if error injection message has not taken effect after 1 minute, the test will timeout
-    private final static long ERROR_INJECTION_MESSAGE_EFFECT_TIMEOUT_MILLISECONDS = 60 * 1000;
-    private final static String TEST_ASC_SECURITY_MESSAGE = "{ \"AgentVersion\": \"0.0.1\", "
-            + "\"AgentId\" : \"{4C1B4747-E4C7-4681-B31D-4B39E390E7F8}\", "
-            + "\"MessageSchemaVersion\" : \"1.0\", \"Events\" : "
-            + " { \"EventType\": \"Security\", "
-            + "\"Category\" : \"Periodic\", "
-            + "\"Name\" : \"ListeningPorts\", "
-            + "\"IsEmpty\" : true, "
-            + "\"PayloadSchemaVersion\" : \"1.0\", "
-            + "\"Id\" : \"%s\", "
-            + "\"TimestampLocal\" : \"2012-04-23T18:25:43.511Z\", "
-            + "\"TimestampUTC\" : \"2012-04-23T18:25:43.511Z\" }, "
-            + "\"Payload\": { \"data\": \"test\" } } }";
-
     private static final int TIMEOUT_MILLISECONDS = 60 * 1000; // 1 minute
     private static final int CHECK_INTERVAL_MILLISECONDS = 300;
 
-    /*
-     * method to send message over given DeviceClient
-     */
-    public static void sendMessages(InternalClient client,
-                                    IotHubClientProtocol protocol,
-                                    List<MessageAndResult> messagesToSend,
-                                    final long RETRY_MILLISECONDS,
-                                    final long SEND_TIMEOUT_MILLISECONDS,
-                                    long interMessageDelay,
-                                    List<Pair<IotHubConnectionStatus, Throwable>> statusUpdates) throws IOException, InterruptedException, IotHubClientException
-    {
-        try
-        {
-            client.open(false);
-
-            for (MessageAndResult messageAndResult : messagesToSend)
-            {
-                if ((protocol == IotHubClientProtocol.MQTT || protocol == IotHubClientProtocol.MQTT_WS) && isErrorInjectionMessage(messageAndResult))
-                {
-                    // error injection message will not be ack'd by service if sent over MQTT/MQTT_WS, so the SDK's
-                    // retry logic will try to send it again after the connection drops. By setting expiry time,
-                    // we ensure that error injection message isn't resent to service too many times. The message will still likely
-                    // be sent 3 or 4 times causing 3 or 4 disconnections, but the test should recover anyways.
-                    messageAndResult.message.setExpiryTime(1000);
-
-                    // Since the message won't be ack'd, then we don't need to validate the status code when this message's callback is fired
-                    messageAndResult.statusCode = null;
-                }
-
-                sendMessageAndWaitForResponse(client, messageAndResult, protocol);
-
-                if (isErrorInjectionMessage(messageAndResult))
-                {
-                    //wait until error injection message takes affect before sending the next message
-                    long startTime = System.currentTimeMillis();
-                    while (!actualStatusUpdatesContainsStatus(statusUpdates, IotHubConnectionStatus.DISCONNECTED_RETRYING))
-                    {
-                        Thread.sleep(1000);
-
-                        // send the fault injection message again in case it wasn't sent successfully before
-                        sendMessageAndWaitForResponse(client, messageAndResult, protocol);
-
-                        if (System.currentTimeMillis() - startTime > ERROR_INJECTION_MESSAGE_EFFECT_TIMEOUT_MILLISECONDS)
-                        {
-                            Assert.fail(buildExceptionMessage("Sending message over " + protocol + " protocol failed: Error injection message never caused connection to be lost", client));
-                        }
-                    }
-                }
-                else
-                {
-                    Thread.sleep(interMessageDelay);
-                }
-            }
-        }
-        finally
-        {
-            client.close();
-        }
-    }
-
-    /*
-     * method to send message over given DeviceClient
-     */
-    public static void sendBulkMessages(InternalClient client,
-                                    IotHubClientProtocol protocol,
-                                    List<MessageAndResult> messagesToSend,
-                                    final long RETRY_MILLISECONDS,
-                                    final long SEND_TIMEOUT_MILLISECONDS,
-                                    long interMessageDelay,
-                                    List<Pair<IotHubConnectionStatus, Throwable>> statusUpdates) throws IOException, InterruptedException, IotHubClientException
-    {
-        try
-        {
-            client.open(false);
-
-            if (protocol != IotHubClientProtocol.HTTPS)
-            {
-                sendMessages(client, protocol, messagesToSend,RETRY_MILLISECONDS ,SEND_TIMEOUT_MILLISECONDS,interMessageDelay, statusUpdates);
-                return;
-            }
-
-            List<Message> bulkMessages = new ArrayList<>();
-            for (MessageAndResult mar : messagesToSend) {
-                bulkMessages.add(mar.message);
-            }
-
-            BulkMessagesAndResult bulkMessagesAndResult = new BulkMessagesAndResult(bulkMessages, IotHubStatusCode.OK);
-            sendBulkMessagesAndWaitForResponse(client, bulkMessagesAndResult, RETRY_MILLISECONDS, SEND_TIMEOUT_MILLISECONDS, protocol);
-        }
-        finally
-        {
-            client.close();
-        }
-    }
-
-    public static void sendMessagesExpectingConnectionStatusChangeUpdate(InternalClient client,
-                                                                         IotHubClientProtocol protocol,
-                                                                         List<MessageAndResult> messagesToSend,
-                                                                         final long RETRY_MILLISECONDS,
-                                                                         final long SEND_TIMEOUT_MILLISECONDS,
-                                                                         final IotHubConnectionStatus expectedStatus,
-                                                                         int interMessageDelay,
-                                                                         AuthenticationType authType) throws IOException, InterruptedException, IotHubClientException
-    {
-        final List<Pair<IotHubConnectionStatus, Throwable>> actualStatusUpdates = new ArrayList<>();
-        client.setConnectionStatusChangeCallback((context) -> actualStatusUpdates.add(new Pair<>(context.getNewStatus(), context.getCause())), new Object());
-
-        sendMessages(client, protocol, messagesToSend, RETRY_MILLISECONDS, SEND_TIMEOUT_MILLISECONDS, interMessageDelay, actualStatusUpdates);
-
-        Assert.assertTrue(buildExceptionMessage(protocol + ", " + authType + ": Expected connection status update to occur: " + expectedStatus, client), actualStatusUpdatesContainsStatus(actualStatusUpdates, expectedStatus));
-    }
-
-    /*
-     * method to send message over given DeviceClient
-     */
-    public static void sendMessagesMultiplex(InternalClient client,
-                                             IotHubClientProtocol protocol,
-                                             final int NUM_MESSAGES_PER_CONNECTION,
-                                             final long RETRY_MILLISECONDS,
-                                             final long SEND_TIMEOUT_MILLISECONDS)
-    {
-        String messageString = "Java client e2e test message over " + protocol + " protocol";
-        Message msg = new Message(messageString);
-
-        for (int i = 0; i < NUM_MESSAGES_PER_CONNECTION; ++i)
-        {
-            try
-            {
-                Success messageSent = new Success();
-                EventCallback callback = new EventCallback(IotHubStatusCode.OK);
-                client.sendEventAsync(msg, callback, messageSent);
-
-                long startTime = System.currentTimeMillis();
-                while (!messageSent.wasCallbackFired())
-                {
-                    Thread.sleep(RETRY_MILLISECONDS);
-                    if (System.currentTimeMillis() - startTime > SEND_TIMEOUT_MILLISECONDS)
-                    {
-                        Assert.fail(buildExceptionMessage("Timed out waiting for message callback", client));
-                    }
-                }
-
-                if (messageSent.getCallbackStatusCode() != IotHubStatusCode.OK)
-                {
-                    Assert.fail(buildExceptionMessage("Sending message over " + protocol + " protocol failed: expected status code OK but received: " + messageSent.getCallbackStatusCode(), client));
-                }
-            }
-            catch (Exception e)
-            {
-                Assert.fail(buildExceptionMessage("Sending message over " + protocol + " protocol failed: Exception encountered while sending messages: " + e.getMessage(), client));
-            }
-        }
-    }
-
-    public static void sendExpiredMessageExpectingMessageExpiredCallback(InternalClient client,
-                                                                         IotHubClientProtocol protocol,
-                                                                         final long RETRY_MILLISECONDS,
-                                                                         final long SEND_TIMEOUT_MILLISECONDS,
-                                                                         AuthenticationType authType) throws IOException
-    {
-        try
-        {
-            Message expiredMessage = new Message("This message has expired");
-            expiredMessage.setAbsoluteExpiryTime(1); //setting this to 0 causes the message to never expire
-            Success messageSentExpiredCallback = new Success();
-
-            client.open(false);
-            client.sendEventAsync(expiredMessage, new EventCallback(IotHubStatusCode.MESSAGE_EXPIRED), messageSentExpiredCallback);
-
-            long startTime = System.currentTimeMillis();
-            while (!messageSentExpiredCallback.wasCallbackFired())
-            {
-                Thread.sleep(RETRY_MILLISECONDS);
-                if (System.currentTimeMillis() - startTime > SEND_TIMEOUT_MILLISECONDS)
-                {
-                    Assert.fail(buildExceptionMessage(protocol + ", " + authType + ": Timed out waiting for a message callback", client));
-                }
-            }
-
-            client.close();
-
-            if (messageSentExpiredCallback.getCallbackStatusCode() != IotHubStatusCode.MESSAGE_EXPIRED)
-            {
-                Assert.fail(buildExceptionMessage("Sending message over " + protocol + " protocol failed: expected status code MESSAGE_EXPIRED but received: " + messageSentExpiredCallback.getCallbackStatusCode(), client));
-            }
-        }
-        catch (Exception e)
-        {
-            client.close();
-            Assert.fail(buildExceptionMessage("Sending expired message over " + protocol + " protocol failed: Exception encountered while sending message and waiting for MESSAGE_EXPIRED callback: " + e.getMessage(), client));
-        }
-    }
-
-    public static void sendMessagesExpectingUnrecoverableConnectionLossAndTimeout(InternalClient client,
-                                                                                  IotHubClientProtocol protocol,
-                                                                                  Message errorInjectionMessage,
-                                                                                  AuthenticationType authType) throws IOException, InterruptedException, IotHubClientException
-    {
-        final List<Pair<IotHubConnectionStatus, Throwable>> statusUpdates = new ArrayList<>();
-        client.setConnectionStatusChangeCallback((context) -> statusUpdates.add(new Pair<>(context.getNewStatus(), context.getCause())), new Object());
-
-        client.open(false);
-
-        client.sendEventAsync(errorInjectionMessage, new EventCallback(null), new Success());
-
-        long startTime = System.currentTimeMillis();
-        while (!(actualStatusUpdatesContainsStatus(statusUpdates, IotHubConnectionStatus.DISCONNECTED_RETRYING) && actualStatusUpdatesContainsStatus(statusUpdates, IotHubConnectionStatus.DISCONNECTED)))
-        {
-            Thread.sleep(500);
-
-            if (System.currentTimeMillis() - startTime > 30 * 1000)
-            {
-                break;
-            }
-        }
-
-        Assert.assertTrue(buildExceptionMessage(protocol + ", " + authType + ": Expected notification about disconnected but retrying.", client), actualStatusUpdatesContainsStatus(statusUpdates, IotHubConnectionStatus.DISCONNECTED_RETRYING));
-        Assert.assertTrue(buildExceptionMessage(protocol + ", " + authType + ": Expected notification about disconnected.", client), actualStatusUpdatesContainsStatus(statusUpdates, IotHubConnectionStatus.DISCONNECTED));
-
-        client.close();
-    }
-
     public static void sendErrorInjectionMessageAndWaitForResponse(InternalClient client, MessageAndResult messageAndResult, IotHubClientProtocol protocol)
     {
         if (protocol == IotHubClientProtocol.MQTT || protocol == IotHubClientProtocol.MQTT_WS)
@@ -309,50 +72,6 @@ public static void sendMessageAndWaitForResponse(InternalClient client, MessageA
         }
     }
 
-    public static void sendBulkMessagesAndWaitForResponse(InternalClient client, BulkMessagesAndResult messagesAndResults, long RETRY_MILLISECONDS, long SEND_TIMEOUT_MILLISECONDS, IotHubClientProtocol protocol)
-    {
-        try
-        {
-            Success messageSent = new Success();
-            EventCallback callback = new EventCallback(messagesAndResults.statusCode);
-            client.sendEventsAsync(messagesAndResults.messages, callback, messageSent);
-
-            long startTime = System.currentTimeMillis();
-            while (!messageSent.wasCallbackFired())
-            {
-                Thread.sleep(RETRY_MILLISECONDS);
-                if (System.currentTimeMillis() - startTime > SEND_TIMEOUT_MILLISECONDS)
-                {
-                    Assert.fail(buildExceptionMessage("Timed out waiting for a message callback", client));
-                    break;
-                }
-            }
-
-            if (messagesAndResults.statusCode != null && messageSent.getCallbackStatusCode() != messagesAndResults.statusCode)
-            {
-                Assert.fail(buildExceptionMessage("Sending message over " + protocol + " protocol failed: expected " + messagesAndResults.statusCode + " but received " + messageSent.getCallbackStatusCode(), client));
-            }
-        }
-        catch (Exception e)
-        {
-            Assert.fail(buildExceptionMessage("Sending message over " + protocol + " protocol failed: Exception encountered while sending and waiting on a message: " + e.getMessage(), client));
-        }
-    }
-
-    private static boolean isErrorInjectionMessage(MessageAndResult messageAndResult)
-    {
-        MessageProperty[] properties = messageAndResult.message.getProperties();
-        for (MessageProperty property : properties)
-        {
-            if (property.getValue().equals(ErrorInjectionHelper.FaultCloseReason_Boom) || property.getValue().equals(ErrorInjectionHelper.FaultCloseReason_Bye))
-            {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
     public static void waitForStabilizedConnection(List<Pair<IotHubConnectionStatus, Throwable>> actualStatusUpdates, InternalClient client) throws InterruptedException
     {
         //Wait until error injection takes effect

iot-e2e-tests/common/src/test/java/tests/integration/com/microsoft/azure/sdk/iot/helpers/IotHubServicesCommon.java

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-package tests.integration.com.microsoft.azure.sdk.iot.iothub.setup;
+package tests.integration.com.microsoft.azure.sdk.iot.helpers;
 
 import lombok.Getter;