-
-
Notifications
You must be signed in to change notification settings - Fork 747
Can someone help me passing this test using Self healing? Self Healer not Overriding. #4942
Unanswered
vatsal-7781
asked this question in
Q&A
-
I am trying to implement Codecept's self healer into my tests ( Setting up AI Provider , Self Healing Codecept )
First to test this,
in my test cases, i tried finding and filling an input called(input type) username, but in the index.html, it's called email
So I want the self healer to suggest and override the test case with correct suggestion to pass the test, instead of failing it.
After following all the steps to implement self healer and logging the AI suggestions.
I can see that AI Model is producing correct suggestions which is to try input[name='email'] instead of I.fillField({ name: 'username' }, 'fakeuser@example.com'); , but i am failing to implement the logic, such that, it overrides the incorrect test case with the correct one.
here are some files and output for context:
steps_file.js:
'use strict';
const { actor } = require('codeceptjs');
const customConfig = require('./codecept.conf').config;
module.exports = function () {
return actor({
async askGptGeneralPrompt(prompt) {
const messages = [{ role: 'user', content: prompt }];
const response = await customConfig.ai.request(messages);
return response;
}
});
};
static_test.js:
Feature('Static HTML File');
Scenario('should display logo, hero content, and buttons', async ({ I }) => {
// Open local HTML file
I.amOnPage('file:///D:/Code_Files/sample/index.html');
I.see('Create Account', '#signup-btn');
// Check other static content
I.see('MySite', '#site-logo');
I.see('Welcome to Our Platform', '#hero-title');
I.see('Discover amazing features and boost your productivity with our tools. Fast. Easy. Powerful.', '#hero-subtext');
I.see('Begin Exploring', '#get-started-btn');
// AI-healable interactions
I.click('#login-btn');
I.click('#signup-btn');
// Try intentionally wrong field selectors to trigger AI healing
I.fillField({ name: 'username' }, 'fakeuser@example.com'); // this does not exist (actual name=email)
I.fillField({ name: 'password' }, 'securepassword123'); // this one is correct
I.click('#get-started-btn');
});
codecept.conf.js:
/** @type {CodeceptJS.MainConfig} */
require('dotenv').config();
require('./heal')
const { setHeadlessWhen } = require('@codeceptjs/configure');
setHeadlessWhen(process.env.HEADLESS);
exports.config = {
tests: './*_test.js',
output: './output',
helpers: {
Playwright: {
browser: 'chromium',
url: 'http://localhost',
show: true
},
// for visual AI testing
"ResembleHelper": {
"require": "codeceptjs-resemblehelper",
"screenshotFolder": "./tests/output/", // folder where WebDriver saves a screenshot when using I.saveScreenshot method
"baseFolder": "./tests/screenshots/base/", // the folder for base images, which will be used with screenshot for comparison
"diffFolder": "./tests/screenshots/diff/" // the folder where resemble would try to store the difference image
}
},
include: {
I: './steps_file.js'
},
// for custom AI
ai: {
request: async (messages) => {
const OpenAI = require('openai')
const openai = new OpenAI({ apiKey: process.env['OPENAI_API_KEY'] })
try {
const completion = await openai.chat.completions.create({
model: 'gpt-4o-mini',
messages,
});
return completion?.choices[0]?.message?.content;
} catch (error) {
console.error('Error communicating with OpenAI:', error);
throw error;
}
},
},
// for inbuilt AI analysis on Fail
plugins: {
heal: {
enabled: true
},
analyze: {
enabled: true,
// analyze up to 3 failures in detail
analyze: 3,
// group similar failures when 5 or more tests fail
clusterize: 5,
// enable screenshot analysis (requires modal that can analyze screenshots)
vision: false
},
pageInfo: {
enabled: true, // <-- ADD THIS
},
},
name: 'codecept-testcases-ai'
}
heal.js:
const { heal, ai } = require('codeceptjs');
heal.addRecipe('ai', {
priority: 10,
prepare: {
// html: ({ I }) => I.grabHTMLFrom('body'),
html: async ({ I }) => {
const body = await I.grabHTMLFrom('body');
const title = await I.grabTextFrom('title').catch(() => '');
return `<title>${title}</title>\n${body}`;
},
},
suggest: true,
steps: [
'fillField',
],
fn: async args => {
// return ai.healFailedStep(args);
const suggestion = await ai.healFailedStep(args);
console.log('🔍 AI Suggestion:', JSON.stringify(suggestion, null, 2));
return suggestion;
}
});
// clickAndType for form inputs (safe guard added)
heal.addRecipe('clickAndType', {
priority: 1,
steps: ['fillField', 'appendField'],
fn: async ({ step }) => {
if (!step || !step.args || step.args.length < 2) {
console.warn('🛑 Healing failed: Invalid step arguments for clickAndType');
return;
}
const locator = step.args[0];
const text = step.args[1];
return ({ I }) => {
I.click(locator);
I.wait(1);
I.type(text);
};
},
});
index.html:
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Simple Page</title>
<style>
body {
margin: 0;
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background-color: #f4f6f8;
color: #333;
}
/* Navbar */
#navbar {
display: flex;
justify-content: space-between;
align-items: center;
padding: 1rem 2rem;
background-color: #fff;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
}
#navbar .logo {
font-size: 1.5rem;
font-weight: bold;
}
#navbar .nav-buttons {
display: flex;
gap: 1rem;
}
.nav-button {
padding: 0.5rem 1rem;
border: none;
cursor: pointer;
font-size: 1rem;
border-radius: 4px;
transition: background-color 0.3s ease;
}
#login-btn {
background-color: transparent;
color: #333;
border: 1px solid #ccc;
}
#signup-btn {
background-color: #007bff;
color: white;
border: none;
}
#login-btn:hover {
background-color: #e6e6e6;
}
#signup-btn:hover {
background-color: #0056b3;
}
/* Hero Section */
#hero {
display: flex;
flex-direction: column;
align-items: center;
text-align: center;
padding: 4rem 2rem;
background: linear-gradient(to right, #6a11cb, #2575fc);
color: white;
}
#hero-title {
font-size: 3rem;
margin-bottom: 1rem;
}
#hero-subtext {
font-size: 1.25rem;
margin-bottom: 2rem;
max-width: 600px;
}
#get-started-btn {
padding: 0.75rem 1.5rem;
font-size: 1rem;
background-color: #fff;
color: #2575fc;
border: none;
border-radius: 4px;
cursor: pointer;
transition: background-color 0.3s ease;
}
#get-started-btn:hover {
background-color: #e0e0ff;
}
</style>
</head>
<body>
<nav id="navbar">
<div class="logo" id="site-logo">MySite</div>
<div class="nav-buttons">
<button id="login-btn" class="nav-button">Sign In</button>
<button id="signup-btn" class="nav-button">Create Account</button>
</div>
</nav>
<section id="hero">
<h1 id="hero-title">Welcome to Our Platform</h1>
<p id="hero-subtext">Discover amazing features and boost your productivity with our tools. Fast. Easy. Powerful.</p>
<button id="get-started-btn">Begin Exploring</button>
<label>Username</label>
<input name="email" type="text" />
<br/>
<br/>
<label>Password</label>
<input name="password" type="password" />
</section>
</body>
</html>
Output:
Just as it should, it says that 1 Step was healed. But i think it's failing top override it.
Can someone help me passing this test using Self healing?
Beta Was this translation helpful? Give feedback.
All reactions
Replies: 0 comments
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment