feat(headless): supporting standard lifecycle events (#5632)

* refactor(headless): use `WaitStable` for `waitload` action

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(headless): add `getNavigationFunc`

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(headless): add `WaitDOM` action

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(headless): add `WaitFMP` action

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(headless): add `WaitFCP` action

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(headless): add `WaitIdle` action

Signed-off-by: Dwi Siswanto <git@dw1.io>

* refactor(headless): `ActionWaitLoad` waits for `proto.PageLifecycleEventNameLoad`

also rename `Page.WaitLoad` to `Page.WaitStable` method.

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(headless): add `WaitStable` action

Signed-off-by: Dwi Siswanto <git@dw1.io>

* refactor(headless): supporting `duration` arg for `WaitStable` action

Signed-off-by: Dwi Siswanto <git@dw1.io>

* chore: ignore `*.png`

Signed-off-by: Dwi Siswanto <git@dw1.io>

* test(headless): update `TestActionScreenshot*`

call `ActionWaitFMP` instead of `WaitLoad` before take screenshot

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(headless): chained with `Timeout` when `WaitStable`

Signed-off-by: Dwi Siswanto <git@dw1.io>

---------

Signed-off-by: Dwi Siswanto <git@dw1.io>

.gitignore

@@ -40,3 +40,6 @@ pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
 pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
 pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
 vendor
+
+# Headless `screenshot` action
+*.png

pkg/protocols/headless/engine/action_types.go

@@ -45,9 +45,24 @@ const (
 	// ActionFilesInput performs an action on a file input.
 	// name:files
 	ActionFilesInput
-	// ActionWaitLoad waits for the page to stop loading.
+	// ActionWaitDOM waits for the HTML document has been completely loaded & parsed.
+	// name:waitdom
+	ActionWaitDOM
+	// ActionWaitFCP waits for the first piece of content (text, image, etc.) is painted on the screen.
+	// name:waitfcp
+	ActionWaitFCP
+	// ActionWaitFMP waits for page has rendered enough meaningful content to be useful to the user.
+	// name:waitfmp
+	ActionWaitFMP
+	// ActionWaitIdle waits for the network is completely idle (no ongoing network requests).
+	// name:waitidle
+	ActionWaitIdle
+	// ActionWaitLoad waits for the page and all its resources (like stylesheets and images) have finished loading.
 	// name:waitload
 	ActionWaitLoad
+	// ActionWaitStable waits until the page is stable.
+	// name:waitstable
+	ActionWaitStable
 	// ActionGetResource performs a get resource action on an element
 	// name:getresource
 	ActionGetResource
@@ -102,7 +117,12 @@ var ActionStringToAction = map[string]ActionType{
 	"time":         ActionTimeInput,
 	"select":       ActionSelectInput,
 	"files":        ActionFilesInput,
+	"waitdom":      ActionWaitDOM,
+	"waitfcp":      ActionWaitFCP,
+	"waitfmp":      ActionWaitFMP,
+	"waitidle":     ActionWaitIdle,
 	"waitload":     ActionWaitLoad,
+	"waitstable":   ActionWaitStable,
 	"getresource":  ActionGetResource,
 	"extract":      ActionExtract,
 	"setmethod":    ActionSetMethod,
@@ -129,7 +149,12 @@ var ActionToActionString = map[ActionType]string{
 	ActionTimeInput:    "time",
 	ActionSelectInput:  "select",
 	ActionFilesInput:   "files",
+	ActionWaitDOM:      "waitdom",
+	ActionWaitFCP:      "waitfcp",
+	ActionWaitFMP:      "waitfmp",
+	ActionWaitIdle:     "waitidle",
 	ActionWaitLoad:     "waitload",
+	ActionWaitStable:   "waitstable",
 	ActionGetResource:  "getresource",
 	ActionExtract:      "extract",
 	ActionSetMethod:    "setmethod",

pkg/protocols/headless/engine/page_actions.go

@@ -94,8 +94,28 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var
 			err = p.TimeInputElement(act, outData)
 		case ActionSelectInput:
 			err = p.SelectInputElement(act, outData)
+		case ActionWaitDOM:
+			event := proto.PageLifecycleEventNameDOMContentLoaded
+			err = p.WaitPageLifecycleEvent(act, outData, event)
+		case ActionWaitFCP:
+			event := proto.PageLifecycleEventNameFirstContentfulPaint
+			err = p.WaitPageLifecycleEvent(act, outData, event)
+		case ActionWaitFMP:
+			event := proto.PageLifecycleEventNameFirstMeaningfulPaint
+			err = p.WaitPageLifecycleEvent(act, outData, event)
+		case ActionWaitIdle:
+			event := proto.PageLifecycleEventNameNetworkIdle
+			err = p.WaitPageLifecycleEvent(act, outData, event)
 		case ActionWaitLoad:
-			err = p.WaitLoad(act, outData)
+			event := proto.PageLifecycleEventNameLoad
+			err = p.WaitPageLifecycleEvent(act, outData, event)
+		case ActionWaitStable:
+			err = p.WaitStable(act, outData)
+		// NOTE(dwisiswant0): Mapping `ActionWaitLoad` to `Page.WaitStable`,
+		// just in case waiting for the `proto.PageLifecycleEventNameLoad` event
+		// doesn't meet expectations.
+		// case ActionWaitLoad, ActionWaitStable:
+		// 	err = p.WaitStable(act, outData)
 		case ActionGetResource:
 			err = p.GetResource(act, outData)
 		case ActionExtract:
@@ -204,6 +224,17 @@ func createBackOffSleeper(pollTimeout, timeout time.Duration) utils.Sleeper {
 	}
 }
 
+func getNavigationFunc(p *Page, act *Action, event proto.PageLifecycleEventName) (func(), error) {
+	dur, err := getTimeout(p, act)
+	if err != nil {
+		return nil, errors.Wrap(err, "Wrong timeout given")
+	}
+
+	fn := p.page.Timeout(dur).WaitNavigation(event)
+
+	return fn, nil
+}
+
 func getTimeout(p *Page, act *Action) (time.Duration, error) {
 	return geTimeParameter(p, act, "timeout", 3, time.Second)
 }
@@ -518,20 +549,38 @@ func (p *Page) SelectInputElement(act *Action, out ActionData) error {
 	return nil
 }
 
-// WaitLoad waits for the page to load
+// WaitPageLifecycleEvent waits for specified page lifecycle event name
-func (p *Page) WaitLoad(act *Action, out ActionData) error {
+func (p *Page) WaitPageLifecycleEvent(act *Action, out ActionData, event proto.PageLifecycleEventName) error {
-	p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
+	fn, err := getNavigationFunc(p, act, event)
-
+	if err != nil {
-	// Wait for the window.onload event and also wait for the network requests
+		return err
-	// to become idle for a maximum duration of 3 seconds. If the requests
-	// do not finish,
-	if err := p.page.WaitLoad(); err != nil {
-		return errors.Wrap(err, "could not wait load event")
 	}
-	_ = p.page.WaitIdle(1 * time.Second)
+
+	fn()
+
 	return nil
 }
 
+// WaitStable waits until the page is stable
+func (p *Page) WaitStable(act *Action, out ActionData) error {
+	var dur time.Duration = time.Second // default stable page duration: 1s
+
+	timeout, err := getTimeout(p, act)
+	if err != nil {
+		return errors.Wrap(err, "Wrong timeout given")
+	}
+
+	argDur := act.Data["duration"]
+	if argDur != "" {
+		dur, err = time.ParseDuration(argDur)
+		if err != nil {
+			dur = time.Second
+		}
+	}
+
+	return p.page.Timeout(timeout).WaitStable(dur)
+}
+
 // GetResource gets a resource from an element from page.
 func (p *Page) GetResource(act *Action, out ActionData) error {
 	element, err := p.pageElementBy(act.Data)

pkg/protocols/headless/engine/page_actions_test.go

@@ -201,7 +201,7 @@ func TestActionScreenshot(t *testing.T) {
 	filePath := filepath.Join(os.TempDir(), "test.png")
 	actions := []*Action{
 		{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
-		{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
+		{ActionType: ActionTypeHolder{ActionType: ActionWaitFMP}},
 		{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath}},
 	}
 
@@ -229,7 +229,7 @@ func TestActionScreenshotToDir(t *testing.T) {
 
 	actions := []*Action{
 		{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
-		{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
+		{ActionType: ActionTypeHolder{ActionType: ActionWaitFMP}},
 		{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath, "mkdir": "true"}},
 	}