From 33f26c8da236761a41b874421ba07979ab1d5bbf Mon Sep 17 00:00:00 2001 From: Julius Härtl Date: Wed, 14 Feb 2018 15:04:08 +0100 Subject: Frontend regression testing with puppeteer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- tests/ui-regression/config.js | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/ui-regression/config.js (limited to 'tests/ui-regression/config.js') diff --git a/tests/ui-regression/config.js b/tests/ui-regression/config.js new file mode 100644 index 00000000000..c6519ba289d --- /dev/null +++ b/tests/ui-regression/config.js @@ -0,0 +1,57 @@ +/** + * @copyright 2018 Julius Härtl + * + * @author 2018 Julius Härtl + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +module.exports = { + + /** + * Define resolutions to be tested when diffing screenshots + */ + resolutions: [ + {title: 'mobile', w: 360, h: 480}, + {title: 'narrow', w: 800, h: 600}, + {title: 'normal', w: 1024, h: 768}, + {title: 'wide', w: 1920, h: 1080}, + {title: 'qhd', w: 2560, h: 1440}, + {title: 'uhd', w: 3840, h: 2160}, + ], + + /** + * URL that holds the base branch + */ + urlBase: 'http://ui-regression-php-master/', + + /** + * URL that holds the branch to be diffed + */ + urlChange: 'http://ui-regression-php/', + + /** + * Path to output directory for screenshot files + */ + outputDirectory: 'out', + + /** + * Run in headless mode (useful for debugging) + */ + headless: true, + +}; -- cgit v1.2.3 From f7d2cdbd978d9af0fe53dc708b10bbf47670aecd Mon Sep 17 00:00:00 2001 From: Julius Härtl Date: Mon, 16 Jul 2018 22:06:44 +0200 Subject: Fix new settings and login tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- tests/ui-regression/config.js | 2 ++ tests/ui-regression/helper.js | 7 +++++-- tests/ui-regression/package.json | 6 +++--- tests/ui-regression/test/appsSpec.js | 2 +- tests/ui-regression/test/installSpec.js | 4 ++-- tests/ui-regression/test/loginSpec.js | 8 +++++++- tests/ui-regression/test/publicSpec.js | 8 ++++---- tests/ui-regression/test/settingsSpec.js | 8 ++++---- 8 files changed, 28 insertions(+), 17 deletions(-) (limited to 'tests/ui-regression/config.js') diff --git a/tests/ui-regression/config.js b/tests/ui-regression/config.js index c6519ba289d..bb0f20240b8 100644 --- a/tests/ui-regression/config.js +++ b/tests/ui-regression/config.js @@ -54,4 +54,6 @@ module.exports = { */ headless: true, + slowMo: 0, + }; diff --git a/tests/ui-regression/helper.js b/tests/ui-regression/helper.js index 7168c80585b..ed2a58ea70a 100644 --- a/tests/ui-regression/helper.js +++ b/tests/ui-regression/helper.js @@ -47,7 +47,8 @@ module.exports = { } this.browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'], - headless: config.headless + headless: config.headless, + slowMo: config.slowMo, }); this.pageBase = await this.browser.newPage(); this.pageCompare = await this.browser.newPage(); @@ -65,6 +66,7 @@ module.exports = { }, performLogin: async function (page, baseUrl) { + await page.bringToFront(); await page.goto(baseUrl + '/index.php/login', {waitUntil: 'networkidle0'}); await page.type('#user', 'admin'); await page.type('#password', 'admin'); @@ -96,6 +98,7 @@ module.exports = { deviceScaleFactor: options.viewport.scale }) ]); + await this.delay(100); } let fileName = test.test.title if (route !== undefined) { @@ -113,7 +116,7 @@ module.exports = { } catch (err) { failed = err; } - await this.delay(500); + await this.delay(100); await Promise.all([ this.pageBase.screenshot({ path: `${this._outputDirectory}/${fileName}.base.png`, diff --git a/tests/ui-regression/package.json b/tests/ui-regression/package.json index 979dfed3dec..8ab4cf530f2 100644 --- a/tests/ui-regression/package.json +++ b/tests/ui-regression/package.json @@ -8,14 +8,14 @@ }, "author": "", "dependencies": { - "fs": "0.0.1-security", "chai": "^4.1.2", - "mocha": "^5.0.0", + "fs": "0.0.1-security", + "mocha": "^5.2.0", "mocha-json-report": "0.0.2", "pixelmatch": "^4.0.2", "png-js": "^0.1.1", "pngjs2": "^2.0.0", "polyserve": "^0.23.0", - "puppeteer": "^1.0.0" + "puppeteer": "^1.6.1" } } diff --git a/tests/ui-regression/test/appsSpec.js b/tests/ui-regression/test/appsSpec.js index b5e5a889e41..3a23c1df773 100644 --- a/tests/ui-regression/test/appsSpec.js +++ b/tests/ui-regression/test/appsSpec.js @@ -39,7 +39,7 @@ describe('apps', function () { }, {viewport: resolution, waitUntil: 'networkidle2'}); }); - ['installed', 'enabled', 'disabled', 'app-bundles'].forEach(function(endpoint) { + ['your-apps', 'enabled', 'disabled', 'app-bundles'].forEach(function(endpoint) { it('apps.' + endpoint + '.' + resolution.title, async function () { return helper.takeAndCompare(this, undefined, async function (page) { try { diff --git a/tests/ui-regression/test/installSpec.js b/tests/ui-regression/test/installSpec.js index 02577883aae..bfb063bd467 100644 --- a/tests/ui-regression/test/installSpec.js +++ b/tests/ui-regression/test/installSpec.js @@ -31,7 +31,7 @@ describe('install', function () { config.resolutions.forEach(function (resolution) { it('show-page.' + resolution.title, async function () { // (test, route, prepare, action, options - return helper.takeAndCompare(this, '/index.php', async (page) => { + return helper.takeAndCompare(this, 'index.php', async (page) => { await helper.delay(100); await page.$eval('body', function (e) { $('#adminlogin').blur(); @@ -61,7 +61,7 @@ describe('install', function () { helper.pageBase.setDefaultNavigationTimeout(5*60*1000); helper.pageCompare.setDefaultNavigationTimeout(5*60*1000); // just run for one resolution since we can only install once - return helper.takeAndCompare(this, '/index.php', async function (page) { + return helper.takeAndCompare(this, 'index.php', async function (page) { const login = await page.type('#adminlogin', 'admin'); const password = await page.type('#adminpass', 'admin'); const inputElement = await page.$('input[type=submit]'); diff --git a/tests/ui-regression/test/loginSpec.js b/tests/ui-regression/test/loginSpec.js index 23f86737a76..8607bbabc84 100644 --- a/tests/ui-regression/test/loginSpec.js +++ b/tests/ui-regression/test/loginSpec.js @@ -62,11 +62,17 @@ describe('login', function () { this.timeout(30000); await helper.resetBrowser(); return helper.takeAndCompare(this, '/', async function (page) { + await page.waitForSelector('input#user'); await page.type('#user', 'admin'); await page.type('#password', 'admin'); const inputElement = await page.$('input[type=submit]'); await inputElement.click(); - await page.waitForNavigation({waitUntil: 'networkidle0'}); + await page.waitForNavigation({waitUntil: 'networkidle2'}); + await page.waitForSelector('#header'); + await page.$eval('body', function (e) { + // force relative timestamp to fixed value, since it breaks screenshot diffing + $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); + }); return await helper.delay(100); }, {viewport: resolution}); }) diff --git a/tests/ui-regression/test/publicSpec.js b/tests/ui-regression/test/publicSpec.js index 843f8f50cef..0893adf9a42 100644 --- a/tests/ui-regression/test/publicSpec.js +++ b/tests/ui-regression/test/publicSpec.js @@ -37,7 +37,7 @@ describe('public', function () { */ config.resolutions.forEach(function (resolution) { it('file-share-invalid.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/s/invalid', async function () { + return helper.takeAndCompare(this, 'index.php/s/invalid', async function () { }, {waitUntil: 'networkidle2', viewport: resolution}); }); }); @@ -48,7 +48,7 @@ describe('public', function () { var shareLink = {}; it('file-share-link', async function () { - return helper.takeAndCompare(this, '/index.php/apps/files', async function (page) { + return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) { const element = await page.$('[data-file="welcome.txt"] .action-share'); await element.click('[data-file="welcome.txt"] .action-share'); await page.waitForSelector('input.linkCheckbox'); @@ -71,7 +71,7 @@ describe('public', function () { config.resolutions.forEach(function (resolution) { it('file-share-valid.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/apps/files', async function (page) { + return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) { await page.goto(shareLink[page.url()]); await helper.delay(500); }, {waitUntil: 'networkidle2', viewport: resolution}); @@ -89,7 +89,7 @@ describe('public', function () { }); it('file-unshare', async function () { - return helper.takeAndCompare(this, '/index.php/apps/files', async function (page) { + return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) { const element = await page.$('[data-file="welcome.txt"] .action-share'); await element.click('[data-file="welcome.txt"] .action-share'); await page.waitForSelector('input.linkCheckbox'); diff --git a/tests/ui-regression/test/settingsSpec.js b/tests/ui-regression/test/settingsSpec.js index 560218c80f8..8b10b281fb6 100644 --- a/tests/ui-regression/test/settingsSpec.js +++ b/tests/ui-regression/test/settingsSpec.js @@ -33,24 +33,24 @@ describe('settings', function () { config.resolutions.forEach(function (resolution) { it('personal.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/settings/user', async function (page) { + return helper.takeAndCompare(this, 'index.php/settings/user', async function (page) { }, {viewport: resolution}); }); it('admin.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/settings/admin', async function (page) { + return helper.takeAndCompare(this, 'index.php/settings/admin', async function (page) { }, {viewport: resolution}); }); ['sharing', 'security', 'theming', 'encryption', 'additional', 'tips-tricks'].forEach(function(endpoint) { it('admin.' + endpoint + '.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/settings/admin/' + endpoint, async function (page) { + return helper.takeAndCompare(this, 'index.php/settings/admin/' + endpoint, async function (page) { }, {viewport: resolution, waitUntil: 'networkidle2'}); }); }); it('usermanagement.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/settings/users', async function (page) { + return helper.takeAndCompare(this, 'index.php/settings/users', async function (page) { }, {viewport: resolution}); }); -- cgit v1.2.3 From 5ef077840f5d0ce24e16404e7cbaf6d8fa25226f Mon Sep 17 00:00:00 2001 From: Julius Härtl Date: Mon, 30 Jul 2018 00:07:16 +0200 Subject: Improve test duration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl Reincrement network idle to 3 s Signed-off-by: Julius Härtl --- tests/ui-regression/config.js | 1 - tests/ui-regression/helper.js | 11 +---------- tests/ui-regression/test/installSpec.js | 8 ++++---- 3 files changed, 5 insertions(+), 15 deletions(-) (limited to 'tests/ui-regression/config.js') diff --git a/tests/ui-regression/config.js b/tests/ui-regression/config.js index bb0f20240b8..c40efd722d7 100644 --- a/tests/ui-regression/config.js +++ b/tests/ui-regression/config.js @@ -31,7 +31,6 @@ module.exports = { {title: 'normal', w: 1024, h: 768}, {title: 'wide', w: 1920, h: 1080}, {title: 'qhd', w: 2560, h: 1440}, - {title: 'uhd', w: 3840, h: 2160}, ], /** diff --git a/tests/ui-regression/helper.js b/tests/ui-regression/helper.js index fd9a074f155..149c4059e99 100644 --- a/tests/ui-regression/helper.js +++ b/tests/ui-regression/helper.js @@ -138,15 +138,6 @@ module.exports = { ]); } - await this.pageBase.$eval('body', function (e) { - // force relative timestamp to fixed value, since it breaks screenshot diffing - $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); - }); - await this.pageCompare.$eval('body', function (e) { - // force relative timestamp to fixed value, since it breaks screenshot diffing - $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); - }); - var failed = null; try { await this.pageBase.bringToFront(); @@ -156,7 +147,6 @@ module.exports = { } catch (err) { failed = err; } - await this.awaitNetworkIdle(3); await this.pageBase.$eval('body', function (e) { $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); $(':focus').blur(); @@ -165,6 +155,7 @@ module.exports = { $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); $(':focus').blur(); }); + await this.awaitNetworkIdle(3); await Promise.all([ this.pageBase.screenshot({ path: `${this._outputDirectory}/${fileName}.base.png`, diff --git a/tests/ui-regression/test/installSpec.js b/tests/ui-regression/test/installSpec.js index d2107cdb61f..ffb4854f1b6 100644 --- a/tests/ui-regression/test/installSpec.js +++ b/tests/ui-regression/test/installSpec.js @@ -44,15 +44,15 @@ describe('install', function () { // (test, route, prepare, action, options return helper.takeAndCompare(this, undefined, async (page) => { await page.click('#showAdvanced'); - await helper.delay(500); - }); + await helper.delay(300); + }, { waitUntil: 'networkidle0', viewport: resolution}); }); it('show-advanced-mysql.' + resolution.title, async function () { // (test, route, prepare, action, options return helper.takeAndCompare(this, undefined, async (page) => { await page.click('label.mysql'); - await helper.delay(500); - }); + await helper.delay(300); + }, { waitUntil: 'networkidle0', viewport: resolution}); }); }); -- cgit v1.2.3