Implement soft assertions

Author: mbroton

playwright/_impl/_assertions.py

@@ -13,7 +13,19 @@
 # limitations under the License.
 
 import collections.abc
-from typing import Any, List, Optional, Pattern, Sequence, Union
+from types import TracebackType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Generic,
+    List,
+    Optional,
+    Pattern,
+    Sequence,
+    Type,
+    TypeVar,
+    Union,
+)
 from urllib.parse import urljoin
 
 from playwright._impl._api_structures import (
@@ -29,6 +41,10 @@
 from playwright._impl._page import Page
 from playwright._impl._str_utils import escape_regex_flags
 
+if TYPE_CHECKING:
+    from ..async_api import Expect as AsyncExpect
+    from ..sync_api import Expect as SyncExpect
+
 
 class AssertionsBase:
     def __init__(
@@ -37,13 +53,15 @@ def __init__(
         timeout: float = None,
         is_not: bool = False,
         message: Optional[str] = None,
+        soft_context: Optional["SoftAssertionContext"] = None,
     ) -> None:
         self._actual_locator = locator
         self._loop = locator._loop
         self._dispatcher_fiber = locator._dispatcher_fiber
         self._timeout = timeout
         self._is_not = is_not
         self._custom_message = message
+        self._soft_context = soft_context
 
     async def _expect_impl(
         self,
@@ -71,9 +89,13 @@ async def _expect_impl(
                 out_message = (
                     f"{message} '{expected}'" if expected is not None else f"{message}"
                 )
-            raise AssertionError(
+            error = AssertionError(
                 f"{out_message}\nActual value: {actual} {format_call_log(result.get('log'))}"
             )
+            if self._soft_context is not None:
+                self._soft_context.add_failure(error)
+            else:
+                raise error
 
 
 class PageAssertions(AssertionsBase):
@@ -83,14 +105,19 @@ def __init__(
         timeout: float = None,
         is_not: bool = False,
         message: Optional[str] = None,
+        soft_context: Optional["SoftAssertionContext"] = None,
     ) -> None:
-        super().__init__(page.locator(":root"), timeout, is_not, message)
+        super().__init__(page.locator(":root"), timeout, is_not, message, soft_context)
         self._actual_page = page
 
     @property
     def _not(self) -> "PageAssertions":
         return PageAssertions(
-            self._actual_page, self._timeout, not self._is_not, self._custom_message
+            self._actual_page,
+            self._timeout,
+            not self._is_not,
+            self._custom_message,
+            self._soft_context,
         )
 
     async def to_have_title(
@@ -148,14 +175,19 @@ def __init__(
         timeout: float = None,
         is_not: bool = False,
         message: Optional[str] = None,
+        soft_context: Optional["SoftAssertionContext"] = None,
     ) -> None:
-        super().__init__(locator, timeout, is_not, message)
+        super().__init__(locator, timeout, is_not, message, soft_context)
         self._actual_locator = locator
 
     @property
     def _not(self) -> "LocatorAssertions":
         return LocatorAssertions(
-            self._actual_locator, self._timeout, not self._is_not, self._custom_message
+            self._actual_locator,
+            self._timeout,
+            not self._is_not,
+            self._custom_message,
+            self._soft_context,
         )
 
     async def to_contain_text(
@@ -848,18 +880,24 @@ def __init__(
         timeout: float = None,
         is_not: bool = False,
         message: Optional[str] = None,
+        soft_context: Optional["SoftAssertionContext"] = None,
     ) -> None:
         self._loop = response._loop
         self._dispatcher_fiber = response._dispatcher_fiber
         self._timeout = timeout
         self._is_not = is_not
         self._actual = response
         self._custom_message = message
+        self._soft_context = soft_context
 
     @property
     def _not(self) -> "APIResponseAssertions":
         return APIResponseAssertions(
-            self._actual, self._timeout, not self._is_not, self._custom_message
+            self._actual,
+            self._timeout,
+            not self._is_not,
+            self._custom_message,
+            self._soft_context,
         )
 
     async def to_be_ok(
@@ -880,7 +918,11 @@ async def to_be_ok(
         if text is not None:
             out_message += f"\n Response Text:\n{text[:1000]}"
 
-        raise AssertionError(out_message)
+        error = AssertionError(out_message)
+        if self._soft_context is not None:
+            self._soft_context.add_failure(error)
+        else:
+            raise error
 
     async def not_to_be_ok(self) -> None:
         __tracebackhide__ = True
@@ -933,3 +975,58 @@ def to_expected_text_values(
         else:
             raise Error("value must be a string or regular expression")
     return out
+
+
+class SoftAssertionContext:
+    def __init__(self) -> None:
+        self._failures: List[Exception] = []
+
+    def __repr__(self) -> str:
+        return f"<SoftAssertionContext failures={self._failures!r}>"
+
+    def add_failure(self, error: Exception) -> None:
+        self._failures.append(error)
+
+    def has_failures(self) -> bool:
+        return bool(self._failures)
+
+    def get_failure_messages(self) -> str:
+        return "\n".join(
+            f"{i}. {str(error)}" for i, error in enumerate(self._failures, 1)
+        )
+
+
+E = TypeVar("E", "SyncExpect", "AsyncExpect")
+
+
+class SoftAssertionContextManager(Generic[E]):
+    def __init__(self, expect: E, context: SoftAssertionContext) -> None:
+        self._expect: E = expect
+        self._context = context
+
+    def __enter__(self) -> E:
+        self._expect._soft_context = self._context
+        return self._expect
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        __tracebackhide__ = True
+
+        if self._context.has_failures():
+            if exc_type is not None:
+                failure_message = (
+                    f"{str(exc_val)}"
+                    f"\n\nThe above exception occurred within soft assertion block."
+                    f"\n\nSoft assertion failures:\n{self._context.get_failure_messages()}"
+                )
+                if exc_val is not None:
+                    exc_val.args = (failure_message,) + exc_val.args[1:]
+                return
+
+            raise AssertionError(
+                f"Soft assertion failures\n{self._context.get_failure_messages()}"
+            )

playwright/async_api/__init__.py

@@ -28,6 +28,10 @@
 )
 from playwright._impl._assertions import LocatorAssertions as LocatorAssertionsImpl
 from playwright._impl._assertions import PageAssertions as PageAssertionsImpl
+from playwright._impl._assertions import (
+    SoftAssertionContext,
+    SoftAssertionContextManager,
+)
 from playwright.async_api._context_manager import PlaywrightContextManager
 from playwright.async_api._generated import (
     Accessibility,
@@ -94,6 +98,7 @@ class Expect:
 
     def __init__(self) -> None:
         self._timeout: Optional[float] = None
+        self._soft_context: Optional[SoftAssertionContext] = None
 
     def set_options(self, timeout: Optional[float] = _unset) -> None:
         """
@@ -108,6 +113,11 @@ def set_options(self, timeout: Optional[float] = _unset) -> None:
         if timeout is not self._unset:
             self._timeout = timeout
 
+    def soft(self) -> SoftAssertionContextManager:
+        expect = Expect()
+        expect._timeout = self._timeout
+        return SoftAssertionContextManager(expect, SoftAssertionContext())
+
     @overload
     def __call__(
         self, actual: Page, message: Optional[str] = None
@@ -128,16 +138,29 @@ def __call__(
     ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]:
         if isinstance(actual, Page):
             return PageAssertions(
-                PageAssertionsImpl(actual._impl_obj, self._timeout, message=message)
+                PageAssertionsImpl(
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    soft_context=self._soft_context,
+                )
             )
         elif isinstance(actual, Locator):
             return LocatorAssertions(
-                LocatorAssertionsImpl(actual._impl_obj, self._timeout, message=message)
+                LocatorAssertionsImpl(
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    soft_context=self._soft_context,
+                )
             )
         elif isinstance(actual, APIResponse):
             return APIResponseAssertions(
                 APIResponseAssertionsImpl(
-                    actual._impl_obj, self._timeout, message=message
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    soft_context=self._soft_context,
                 )
             )
         raise ValueError(f"Unsupported type: {type(actual)}")

playwright/sync_api/__init__.py

@@ -28,6 +28,10 @@
 )
 from playwright._impl._assertions import LocatorAssertions as LocatorAssertionsImpl
 from playwright._impl._assertions import PageAssertions as PageAssertionsImpl
+from playwright._impl._assertions import (
+    SoftAssertionContext,
+    SoftAssertionContextManager,
+)
 from playwright.sync_api._context_manager import PlaywrightContextManager
 from playwright.sync_api._generated import (
     Accessibility,
@@ -94,6 +98,7 @@ class Expect:
 
     def __init__(self) -> None:
         self._timeout: Optional[float] = None
+        self._soft_context: Optional[SoftAssertionContext] = None
 
     def set_options(self, timeout: Optional[float] = _unset) -> None:
         """
@@ -108,6 +113,11 @@ def set_options(self, timeout: Optional[float] = _unset) -> None:
         if timeout is not self._unset:
             self._timeout = timeout
 
+    def soft(self) -> SoftAssertionContextManager:
+        expect = Expect()
+        expect._timeout = self._timeout
+        return SoftAssertionContextManager(expect, SoftAssertionContext())
+
     @overload
     def __call__(
         self, actual: Page, message: Optional[str] = None
@@ -128,16 +138,29 @@ def __call__(
     ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]:
         if isinstance(actual, Page):
             return PageAssertions(
-                PageAssertionsImpl(actual._impl_obj, self._timeout, message=message)
+                PageAssertionsImpl(
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    soft_context=self._soft_context,
+                )
             )
         elif isinstance(actual, Locator):
             return LocatorAssertions(
-                LocatorAssertionsImpl(actual._impl_obj, self._timeout, message=message)
+                LocatorAssertionsImpl(
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    soft_context=self._soft_context,
+                )
             )
         elif isinstance(actual, APIResponse):
             return APIResponseAssertions(
                 APIResponseAssertionsImpl(
-                    actual._impl_obj, self._timeout, message=message
+                    actual._impl_obj,
+                    self._timeout,
+                    message=message,
+                    soft_context=self._soft_context,
                 )
             )
         raise ValueError(f"Unsupported type: {type(actual)}")