Merge changes Ie19647cd,Ib62f4592,Ib7f90d94 into pi-dev
* changes:
[AWARE] Add Latency measurement for a complete L2 end-to-end setup
[AWARE] Update data analysis to extract CDF
[AWARE] Update stress tests with PASS/FAIL critera
diff --git a/acts/framework/acts/test_utils/wifi/aware/aware_test_utils.py b/acts/framework/acts/test_utils/wifi/aware/aware_test_utils.py
index 05ba784..eada097 100644
--- a/acts/framework/acts/test_utils/wifi/aware/aware_test_utils.py
+++ b/acts/framework/acts/test_utils/wifi/aware/aware_test_utils.py
@@ -310,21 +310,78 @@
data_min = min(data)
data_max = max(data)
data_mean = statistics.mean(data)
+ data_cdf = extract_cdf(data)
+ data_cdf_decile = extract_cdf_decile(data_cdf)
results['%smin' % key_prefix] = data_min
results['%smax' % key_prefix] = data_max
results['%smean' % key_prefix] = data_mean
+ results['%scdf' % key_prefix] = data_cdf
+ results['%scdf_decile' % key_prefix] = data_cdf_decile
results['%sraw_data' % key_prefix] = data
if num_samples > 1:
data_stdev = statistics.stdev(data)
results['%sstdev' % key_prefix] = data_stdev
- ad.log.info('%s: num_samples=%d, min=%.2f, max=%.2f, mean=%.2f, stdev=%.2f',
- log_prefix, num_samples, data_min, data_max, data_mean,
- data_stdev)
+ ad.log.info(
+ '%s: num_samples=%d, min=%.2f, max=%.2f, mean=%.2f, stdev=%.2f, cdf_decile=%s',
+ log_prefix, num_samples, data_min, data_max, data_mean, data_stdev,
+ data_cdf_decile)
else:
- ad.log.info('%s: num_samples=%d, min=%.2f, max=%.2f, mean=%.2f', log_prefix,
- num_samples, data_min, data_max, data_mean)
+ ad.log.info(
+ '%s: num_samples=%d, min=%.2f, max=%.2f, mean=%.2f, cdf_decile=%s',
+ log_prefix, num_samples, data_min, data_max, data_mean, data_cdf_decile)
+
+def extract_cdf_decile(cdf):
+ """Extracts the 10%, 20%, ..., 90% points from the CDF and returns their
+ value (a list of 9 values).
+
+ Since CDF may not (will not) have exact x% value picks the value >= x%.
+
+ Args:
+ cdf: a list of 2 lists, the X and Y of the CDF.
+ """
+ decades = []
+ next_decade = 10
+ for x, y in zip(cdf[0], cdf[1]):
+ while 100*y >= next_decade:
+ decades.append(x)
+ next_decade = next_decade + 10
+ if next_decade == 100:
+ break
+ return decades
+
+def extract_cdf(data):
+ """Calculates the Cumulative Distribution Function (CDF) of the data.
+
+ Args:
+ data: A list containing data (does not have to be sorted).
+
+ Returns: a list of 2 lists: the X and Y axis of the CDF.
+ """
+ x = []
+ cdf = []
+ if not data:
+ return (x, cdf)
+
+ all_values = sorted(data)
+ for val in all_values:
+ if not x:
+ x.append(val)
+ cdf.append(1)
+ else:
+ if x[-1] == val:
+ cdf[-1] += 1
+ else:
+ x.append(val)
+ cdf.append(cdf[-1] + 1)
+
+ scale = 1.0 / len(all_values)
+ for i in range(len(cdf)):
+ cdf[i] = cdf[i] * scale
+
+ return (x, cdf)
+
def get_mac_addr(device, interface):
"""Get the MAC address of the specified interface. Uses ifconfig and parses
diff --git a/acts/tests/google/wifi/aware/performance/LatencyTest.py b/acts/tests/google/wifi/aware/performance/LatencyTest.py
index 9f2a5bf..bfadebc 100644
--- a/acts/tests/google/wifi/aware/performance/LatencyTest.py
+++ b/acts/tests/google/wifi/aware/performance/LatencyTest.py
@@ -427,6 +427,209 @@
dw_5ghz))
results[key_avail]["ndp_setup_failures"] = ndp_setup_failures
+ def run_end_to_end_latency(self, results, dw_24ghz, dw_5ghz, num_iterations,
+ startup_offset, include_setup):
+ """Measure the latency for end-to-end communication link setup:
+ - Start Aware
+ - Discovery
+ - Message from Sub -> Pub
+ - Message from Pub -> Sub
+ - NDP setup
+
+ Args:
+ results: Result array to be populated - will add results (not erase it)
+ dw_24ghz: DW interval in the 2.4GHz band.
+ dw_5ghz: DW interval in the 5GHz band.
+ startup_offset: The start-up gap (in seconds) between the two devices
+ include_setup: True to include the cluster setup in the latency
+ measurements.
+ """
+ key = "dw24_%d_dw5_%d" % (dw_24ghz, dw_5ghz)
+ results[key] = {}
+ results[key]["num_iterations"] = num_iterations
+
+ p_dut = self.android_devices[0]
+ p_dut.pretty_name = "Publisher"
+ s_dut = self.android_devices[1]
+ s_dut.pretty_name = "Subscriber"
+
+ # override the default DW configuration
+ autils.config_power_settings(p_dut, dw_24ghz, dw_5ghz)
+ autils.config_power_settings(s_dut, dw_24ghz, dw_5ghz)
+
+ latencies = []
+
+ # allow for failures here since running lots of samples and would like to
+ # get the partial data even in the presence of errors
+ failures = 0
+
+ if not include_setup:
+ # Publisher+Subscriber: attach and wait for confirmation
+ p_id = p_dut.droid.wifiAwareAttach(False)
+ autils.wait_for_event(p_dut, aconsts.EVENT_CB_ON_ATTACHED)
+ time.sleep(startup_offset)
+ s_id = s_dut.droid.wifiAwareAttach(False)
+ autils.wait_for_event(s_dut, aconsts.EVENT_CB_ON_ATTACHED)
+
+ for i in range(num_iterations):
+ while (True): # for pseudo-goto/finalize
+ timestamp_start = time.perf_counter()
+
+ if include_setup:
+ # Publisher+Subscriber: attach and wait for confirmation
+ p_id = p_dut.droid.wifiAwareAttach(False)
+ autils.wait_for_event(p_dut, aconsts.EVENT_CB_ON_ATTACHED)
+ time.sleep(startup_offset)
+ s_id = s_dut.droid.wifiAwareAttach(False)
+ autils.wait_for_event(s_dut, aconsts.EVENT_CB_ON_ATTACHED)
+
+ # start publish
+ p_disc_id, p_disc_event = self.start_discovery_session(
+ p_dut, p_id, True, aconsts.PUBLISH_TYPE_UNSOLICITED)
+
+ # start subscribe
+ s_disc_id, s_session_event = self.start_discovery_session(
+ s_dut, s_id, False, aconsts.SUBSCRIBE_TYPE_PASSIVE)
+
+ # wait for discovery (allow for failures here since running lots of
+ # samples and would like to get the partial data even in the presence of
+ # errors)
+ try:
+ event = s_dut.ed.pop_event(aconsts.SESSION_CB_ON_SERVICE_DISCOVERED,
+ autils.EVENT_TIMEOUT)
+ s_dut.log.info("[Subscriber] SESSION_CB_ON_SERVICE_DISCOVERED: %s",
+ event["data"])
+ peer_id_on_sub = event['data'][aconsts.SESSION_CB_KEY_PEER_ID]
+ except queue.Empty:
+ s_dut.log.info("[Subscriber] Timed out while waiting for "
+ "SESSION_CB_ON_SERVICE_DISCOVERED")
+ failures = failures + 1
+ break
+
+ # message from Sub -> Pub
+ msg_s2p = "Message Subscriber -> Publisher #%d" % i
+ next_msg_id = self.get_next_msg_id()
+ s_dut.droid.wifiAwareSendMessage(s_disc_id, peer_id_on_sub, next_msg_id,
+ msg_s2p, 0)
+
+ # wait for Tx confirmation
+ try:
+ s_dut.ed.pop_event(aconsts.SESSION_CB_ON_MESSAGE_SENT,
+ autils.EVENT_TIMEOUT)
+ except queue.Empty:
+ s_dut.log.info("[Subscriber] Timed out while waiting for "
+ "SESSION_CB_ON_MESSAGE_SENT")
+ failures = failures + 1
+ break
+
+ # wait for Rx confirmation (and validate contents)
+ try:
+ event = p_dut.ed.pop_event(aconsts.SESSION_CB_ON_MESSAGE_RECEIVED,
+ autils.EVENT_TIMEOUT)
+ peer_id_on_pub = event['data'][aconsts.SESSION_CB_KEY_PEER_ID]
+ if (event["data"][
+ aconsts.SESSION_CB_KEY_MESSAGE_AS_STRING] != msg_s2p):
+ p_dut.log.info("[Publisher] Corrupted input message - %s", event)
+ failures = failures + 1
+ break
+ except queue.Empty:
+ p_dut.log.info("[Publisher] Timed out while waiting for "
+ "SESSION_CB_ON_MESSAGE_RECEIVED")
+ failures = failures + 1
+ break
+
+ # message from Pub -> Sub
+ msg_p2s = "Message Publisher -> Subscriber #%d" % i
+ next_msg_id = self.get_next_msg_id()
+ p_dut.droid.wifiAwareSendMessage(p_disc_id, peer_id_on_pub, next_msg_id,
+ msg_p2s, 0)
+
+ # wait for Tx confirmation
+ try:
+ p_dut.ed.pop_event(aconsts.SESSION_CB_ON_MESSAGE_SENT,
+ autils.EVENT_TIMEOUT)
+ except queue.Empty:
+ p_dut.log.info("[Publisher] Timed out while waiting for "
+ "SESSION_CB_ON_MESSAGE_SENT")
+ failures = failures + 1
+ break
+
+ # wait for Rx confirmation (and validate contents)
+ try:
+ event = s_dut.ed.pop_event(aconsts.SESSION_CB_ON_MESSAGE_RECEIVED,
+ autils.EVENT_TIMEOUT)
+ if (event["data"][
+ aconsts.SESSION_CB_KEY_MESSAGE_AS_STRING] != msg_p2s):
+ s_dut.log.info("[Subscriber] Corrupted input message - %s", event)
+ failures = failures + 1
+ break
+ except queue.Empty:
+ s_dut.log.info("[Subscriber] Timed out while waiting for "
+ "SESSION_CB_ON_MESSAGE_RECEIVED")
+ failures = failures + 1
+ break
+
+ # create NDP
+
+ # Publisher: request network
+ p_req_key = autils.request_network(
+ p_dut,
+ p_dut.droid.wifiAwareCreateNetworkSpecifier(p_disc_id,
+ peer_id_on_pub, None))
+
+ # Subscriber: request network
+ s_req_key = autils.request_network(
+ s_dut,
+ s_dut.droid.wifiAwareCreateNetworkSpecifier(s_disc_id,
+ peer_id_on_sub, None))
+
+ # Publisher & Subscriber: wait for network formation
+ try:
+ p_net_event = autils.wait_for_event_with_keys(
+ p_dut, cconsts.EVENT_NETWORK_CALLBACK, autils.EVENT_TIMEOUT, (
+ cconsts.NETWORK_CB_KEY_EVENT,
+ cconsts.NETWORK_CB_LINK_PROPERTIES_CHANGED),
+ (cconsts.NETWORK_CB_KEY_ID, p_req_key))
+ s_net_event = autils.wait_for_event_with_keys(
+ s_dut, cconsts.EVENT_NETWORK_CALLBACK, autils.EVENT_TIMEOUT, (
+ cconsts.NETWORK_CB_KEY_EVENT,
+ cconsts.NETWORK_CB_LINK_PROPERTIES_CHANGED),
+ (cconsts.NETWORK_CB_KEY_ID, s_req_key))
+ except:
+ failures = failures + 1
+ break
+
+ p_aware_if = p_net_event["data"][cconsts.NETWORK_CB_KEY_INTERFACE_NAME]
+ s_aware_if = s_net_event["data"][cconsts.NETWORK_CB_KEY_INTERFACE_NAME]
+
+ p_ipv6 = \
+ p_dut.droid.connectivityGetLinkLocalIpv6Address(p_aware_if).split("%")[
+ 0]
+ s_ipv6 = \
+ s_dut.droid.connectivityGetLinkLocalIpv6Address(s_aware_if).split("%")[
+ 0]
+
+ p_dut.log.info("[Publisher] IF=%s, IPv6=%s", p_aware_if, p_ipv6)
+ s_dut.log.info("[Subscriber] IF=%s, IPv6=%s", s_aware_if, s_ipv6)
+
+ latencies.append(time.perf_counter() - timestamp_start)
+ break
+
+ # destroy sessions
+ p_dut.droid.wifiAwareDestroyDiscoverySession(p_disc_id)
+ s_dut.droid.wifiAwareDestroyDiscoverySession(s_disc_id)
+ if include_setup:
+ p_dut.droid.wifiAwareDestroy(p_id)
+ s_dut.droid.wifiAwareDestroy(s_id)
+
+ autils.extract_stats(
+ p_dut,
+ data=latencies,
+ results=results[key],
+ key_prefix="",
+ log_prefix="End-to-End(dw24=%d, dw5=%d)" % (dw_24ghz, dw_5ghz))
+ results[key]["failures"] = failures
+
########################################################################
@@ -554,3 +757,44 @@
num_iterations=100)
asserts.explicit_pass(
"test_ndp_setup_latency_non_interactive_dws finished", extras=results)
+
+ def test_end_to_end_latency_default_dws(self):
+ """Measure the latency for end-to-end communication link setup:
+ - Start Aware
+ - Discovery
+ - Message from Sub -> Pub
+ - Message from Pub -> Sub
+ - NDP setup
+ """
+ results = {}
+ self.run_end_to_end_latency(
+ results,
+ dw_24ghz=aconsts.POWER_DW_24_INTERACTIVE,
+ dw_5ghz=aconsts.POWER_DW_5_INTERACTIVE,
+ num_iterations=10,
+ startup_offset=0,
+ include_setup=True)
+ asserts.explicit_pass(
+ "test_end_to_end_latency_default_dws finished", extras=results)
+
+ def test_end_to_end_latency_post_attach_default_dws(self):
+ """Measure the latency for end-to-end communication link setup without
+ the initial synchronization:
+ - Start Aware & synchronize initially
+ - Loop:
+ - Discovery
+ - Message from Sub -> Pub
+ - Message from Pub -> Sub
+ - NDP setup
+ """
+ results = {}
+ self.run_end_to_end_latency(
+ results,
+ dw_24ghz=aconsts.POWER_DW_24_INTERACTIVE,
+ dw_5ghz=aconsts.POWER_DW_5_INTERACTIVE,
+ num_iterations=10,
+ startup_offset=0,
+ include_setup=False)
+ asserts.explicit_pass(
+ "test_end_to_end_latency_post_attach_default_dws finished",
+ extras=results)
diff --git a/acts/tests/google/wifi/aware/stress/DataPathStressTest.py b/acts/tests/google/wifi/aware/stress/DataPathStressTest.py
index 28de5a7..ae0564e 100644
--- a/acts/tests/google/wifi/aware/stress/DataPathStressTest.py
+++ b/acts/tests/google/wifi/aware/stress/DataPathStressTest.py
@@ -30,7 +30,10 @@
ATTACH_ITERATIONS = 2
# Number of iterations on create/destroy NDP in each discovery session.
- NDP_ITERATIONS = 20
+ NDP_ITERATIONS = 50
+
+ # Maximum percentage of NDP setup failures over all iterations
+ MAX_FAILURE_PERCENTAGE = 1
def __init__(self, controllers):
AwareBaseTest.__init__(self, controllers)
@@ -87,9 +90,9 @@
resp_id, aconsts.DATA_PATH_RESPONDER, init_mac, None))
# Wait a minimal amount of time to let the Responder configure itself
- # and be ready for the request. While calling it first may be sufficient
- # there are no guarantees that a glitch may slow the Responder slightly
- # enough to invert the setup order.
+ # and be ready for the request. While calling it first may be
+ # sufficient there are no guarantees that a glitch may slow the
+ # Responder slightly enough to invert the setup order.
time.sleep(1)
# Initiator: request network
@@ -106,7 +109,7 @@
# Wait a minimal amount of time to let the Initiator configure itself
# to guarantee failure!
- time.sleep(1)
+ time.sleep(2)
# Responder: request network
resp_req_key = autils.request_network(
@@ -171,12 +174,16 @@
results['ndp_init_setup_failures'] = ndp_init_setup_failures
results['ndp_resp_setup_success'] = ndp_resp_setup_success
results['ndp_resp_setup_failures'] = ndp_resp_setup_failures
- asserts.assert_equal(
- ndp_init_setup_failures + ndp_resp_setup_failures,
- 0,
- 'test_oob_ndp_stress finished',
- extras=results)
- asserts.explicit_pass("test_oob_ndp_stress done", extras=results)
+ max_failures = (
+ self.MAX_FAILURE_PERCENTAGE * attach_iterations * ndp_iterations / 100)
+ if max_failures == 0:
+ max_failures = 1
+ if trigger_failure_on_index is not None:
+ max_failures = max_failures + 1 # for the triggered failure
+ asserts.assert_true(
+ (ndp_init_setup_failures + ndp_resp_setup_failures) < (2 * max_failures),
+ 'NDP setup failure rate exceeds threshold', extras=results)
+ asserts.explicit_pass("test_oob_ndp_stress* done", extras=results)
def test_oob_ndp_stress(self):
"""Run NDP (NAN data-path) stress test creating and destroying Aware
@@ -190,4 +197,6 @@
Verify recovery from failure by triggering an artifical failure and
verifying that all subsequent iterations succeed.
"""
- self.run_oob_ndp_stress(1, 3, 0)
+ self.run_oob_ndp_stress(attach_iterations=1,
+ ndp_iterations=10,
+ trigger_failure_on_index=3)
diff --git a/acts/tests/google/wifi/aware/stress/DiscoveryStressTest.py b/acts/tests/google/wifi/aware/stress/DiscoveryStressTest.py
index 8b3d925..730df18 100644
--- a/acts/tests/google/wifi/aware/stress/DiscoveryStressTest.py
+++ b/acts/tests/google/wifi/aware/stress/DiscoveryStressTest.py
@@ -15,7 +15,6 @@
# limitations under the License.
import queue
-import time
from acts import asserts
from acts.test_utils.wifi.aware import aware_const as aconsts
@@ -101,9 +100,6 @@
results = {}
results['discovery_setup_success'] = discovery_setup_success
results['discovery_setup_fail'] = discovery_setup_fail
- asserts.assert_equal(
- discovery_setup_fail,
- 0,
- 'test_discovery_stress finished',
- extras=results)
+ asserts.assert_equal(discovery_setup_fail, 0,
+ 'Discovery setup failures', extras=results)
asserts.explicit_pass('test_discovery_stress done', extras=results)
diff --git a/acts/tests/google/wifi/aware/stress/MessagesStressTest.py b/acts/tests/google/wifi/aware/stress/MessagesStressTest.py
index 5871d61..02b423c 100644
--- a/acts/tests/google/wifi/aware/stress/MessagesStressTest.py
+++ b/acts/tests/google/wifi/aware/stress/MessagesStressTest.py
@@ -29,7 +29,19 @@
class MessagesStressTest(AwareBaseTest):
"""Set of stress tests for Wi-Fi Aware L2 (layer 2) message exchanges."""
+
+ # Number of iterations in the stress test (number of messages)
NUM_ITERATIONS = 100
+
+ # Maximum permitted percentage of messages which fail to be transmitted
+ # correctly
+ MAX_TX_FAILURE_PERCENTAGE = 2
+
+ # Maximum permitted percentage of messages which are received more than once
+ # (indicating, most likely, that the ACK wasn't received and the message was
+ # retransmitted)
+ MAX_DUPLICATE_RX_PERCENTAGE = 2
+
SERVICE_NAME = "GoogleTestServiceXY"
def __init__(self, controllers):
@@ -250,6 +262,9 @@
# clear errors
asserts.assert_equal(results["tx_unknown_ids"], 0, "Message ID corruption",
results)
+ asserts.assert_equal(results["tx_count_neither"], 0,
+ "Tx message with no success or fail indication",
+ results)
asserts.assert_equal(results["tx_count_duplicate_fail"], 0,
"Duplicate Tx fail messages", results)
asserts.assert_equal(results["tx_count_duplicate_success"], 0,
@@ -267,3 +282,13 @@
"Message received but Tx didn't get ACK", results)
asserts.explicit_pass("test_stress_message done", extras=results)
+
+ # permissible failures based on thresholds
+ asserts.assert_true(results["tx_count_fail"] <= (
+ self.MAX_TX_FAILURE_PERCENTAGE * self.NUM_ITERATIONS / 100),
+ "Number of Tx failures exceeds threshold",
+ extras=results)
+ asserts.assert_true(results["rx_count_duplicate"] <= (
+ self.MAX_DUPLICATE_RX_PERCENTAGE * self.NUM_ITERATIONS / 100),
+ "Number of duplicate Rx exceeds threshold",
+ extras=results)