From 33f26c8da236761a41b874421ba07979ab1d5bbf Mon Sep 17 00:00:00 2001 From: Julius Härtl Date: Wed, 14 Feb 2018 15:04:08 +0100 Subject: Frontend regression testing with puppeteer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- tests/ui-regression/test/installSpec.js | 75 +++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/ui-regression/test/installSpec.js (limited to 'tests/ui-regression/test/installSpec.js') diff --git a/tests/ui-regression/test/installSpec.js b/tests/ui-regression/test/installSpec.js new file mode 100644 index 00000000000..02577883aae --- /dev/null +++ b/tests/ui-regression/test/installSpec.js @@ -0,0 +1,75 @@ +/** + * @copyright 2018 Julius Härtl + * + * @author 2018 Julius Härtl + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +const helper = require('../helper.js'); +const config = require('../config.js'); + +describe('install', function () { + + before(async () => await helper.init(this)); + after(async () => await helper.exit()); + + config.resolutions.forEach(function (resolution) { + it('show-page.' + resolution.title, async function () { + // (test, route, prepare, action, options + return helper.takeAndCompare(this, '/index.php', async (page) => { + await helper.delay(100); + await page.$eval('body', function (e) { + $('#adminlogin').blur(); + }); + await helper.delay(100); + }, { waitUntil: 'networkidle0', viewport: resolution}); + }); + + it('show-advanced.' + resolution.title, async function () { + // (test, route, prepare, action, options + return helper.takeAndCompare(this, undefined, async (page) => { + await page.click('#showAdvanced'); + await helper.delay(500); + }); + }); + it('show-advanced-mysql.' + resolution.title, async function () { + // (test, route, prepare, action, options + return helper.takeAndCompare(this, undefined, async (page) => { + await page.click('label.mysql'); + await helper.delay(500); + }); + }); + }); + + it('runs', async function () { + this.timeout(5*60*1000); + helper.pageBase.setDefaultNavigationTimeout(5*60*1000); + helper.pageCompare.setDefaultNavigationTimeout(5*60*1000); + // just run for one resolution since we can only install once + return helper.takeAndCompare(this, '/index.php', async function (page) { + const login = await page.type('#adminlogin', 'admin'); + const password = await page.type('#adminpass', 'admin'); + const inputElement = await page.$('input[type=submit]'); + await inputElement.click(); + await page.waitForNavigation({waitUntil: 'networkidle0'}); + helper.pageBase.setDefaultNavigationTimeout(60000); + helper.pageCompare.setDefaultNavigationTimeout(60000); + }, { waitUntil: 'networkidle0', viewport: {w: 1920, h: 1080}}); + }); + +}); -- cgit v1.2.3 From f7d2cdbd978d9af0fe53dc708b10bbf47670aecd Mon Sep 17 00:00:00 2001 From: Julius Härtl Date: Mon, 16 Jul 2018 22:06:44 +0200 Subject: Fix new settings and login tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- tests/ui-regression/config.js | 2 ++ tests/ui-regression/helper.js | 7 +++++-- tests/ui-regression/package.json | 6 +++--- tests/ui-regression/test/appsSpec.js | 2 +- tests/ui-regression/test/installSpec.js | 4 ++-- tests/ui-regression/test/loginSpec.js | 8 +++++++- tests/ui-regression/test/publicSpec.js | 8 ++++---- tests/ui-regression/test/settingsSpec.js | 8 ++++---- 8 files changed, 28 insertions(+), 17 deletions(-) (limited to 'tests/ui-regression/test/installSpec.js') diff --git a/tests/ui-regression/config.js b/tests/ui-regression/config.js index c6519ba289d..bb0f20240b8 100644 --- a/tests/ui-regression/config.js +++ b/tests/ui-regression/config.js @@ -54,4 +54,6 @@ module.exports = { */ headless: true, + slowMo: 0, + }; diff --git a/tests/ui-regression/helper.js b/tests/ui-regression/helper.js index 7168c80585b..ed2a58ea70a 100644 --- a/tests/ui-regression/helper.js +++ b/tests/ui-regression/helper.js @@ -47,7 +47,8 @@ module.exports = { } this.browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'], - headless: config.headless + headless: config.headless, + slowMo: config.slowMo, }); this.pageBase = await this.browser.newPage(); this.pageCompare = await this.browser.newPage(); @@ -65,6 +66,7 @@ module.exports = { }, performLogin: async function (page, baseUrl) { + await page.bringToFront(); await page.goto(baseUrl + '/index.php/login', {waitUntil: 'networkidle0'}); await page.type('#user', 'admin'); await page.type('#password', 'admin'); @@ -96,6 +98,7 @@ module.exports = { deviceScaleFactor: options.viewport.scale }) ]); + await this.delay(100); } let fileName = test.test.title if (route !== undefined) { @@ -113,7 +116,7 @@ module.exports = { } catch (err) { failed = err; } - await this.delay(500); + await this.delay(100); await Promise.all([ this.pageBase.screenshot({ path: `${this._outputDirectory}/${fileName}.base.png`, diff --git a/tests/ui-regression/package.json b/tests/ui-regression/package.json index 979dfed3dec..8ab4cf530f2 100644 --- a/tests/ui-regression/package.json +++ b/tests/ui-regression/package.json @@ -8,14 +8,14 @@ }, "author": "", "dependencies": { - "fs": "0.0.1-security", "chai": "^4.1.2", - "mocha": "^5.0.0", + "fs": "0.0.1-security", + "mocha": "^5.2.0", "mocha-json-report": "0.0.2", "pixelmatch": "^4.0.2", "png-js": "^0.1.1", "pngjs2": "^2.0.0", "polyserve": "^0.23.0", - "puppeteer": "^1.0.0" + "puppeteer": "^1.6.1" } } diff --git a/tests/ui-regression/test/appsSpec.js b/tests/ui-regression/test/appsSpec.js index b5e5a889e41..3a23c1df773 100644 --- a/tests/ui-regression/test/appsSpec.js +++ b/tests/ui-regression/test/appsSpec.js @@ -39,7 +39,7 @@ describe('apps', function () { }, {viewport: resolution, waitUntil: 'networkidle2'}); }); - ['installed', 'enabled', 'disabled', 'app-bundles'].forEach(function(endpoint) { + ['your-apps', 'enabled', 'disabled', 'app-bundles'].forEach(function(endpoint) { it('apps.' + endpoint + '.' + resolution.title, async function () { return helper.takeAndCompare(this, undefined, async function (page) { try { diff --git a/tests/ui-regression/test/installSpec.js b/tests/ui-regression/test/installSpec.js index 02577883aae..bfb063bd467 100644 --- a/tests/ui-regression/test/installSpec.js +++ b/tests/ui-regression/test/installSpec.js @@ -31,7 +31,7 @@ describe('install', function () { config.resolutions.forEach(function (resolution) { it('show-page.' + resolution.title, async function () { // (test, route, prepare, action, options - return helper.takeAndCompare(this, '/index.php', async (page) => { + return helper.takeAndCompare(this, 'index.php', async (page) => { await helper.delay(100); await page.$eval('body', function (e) { $('#adminlogin').blur(); @@ -61,7 +61,7 @@ describe('install', function () { helper.pageBase.setDefaultNavigationTimeout(5*60*1000); helper.pageCompare.setDefaultNavigationTimeout(5*60*1000); // just run for one resolution since we can only install once - return helper.takeAndCompare(this, '/index.php', async function (page) { + return helper.takeAndCompare(this, 'index.php', async function (page) { const login = await page.type('#adminlogin', 'admin'); const password = await page.type('#adminpass', 'admin'); const inputElement = await page.$('input[type=submit]'); diff --git a/tests/ui-regression/test/loginSpec.js b/tests/ui-regression/test/loginSpec.js index 23f86737a76..8607bbabc84 100644 --- a/tests/ui-regression/test/loginSpec.js +++ b/tests/ui-regression/test/loginSpec.js @@ -62,11 +62,17 @@ describe('login', function () { this.timeout(30000); await helper.resetBrowser(); return helper.takeAndCompare(this, '/', async function (page) { + await page.waitForSelector('input#user'); await page.type('#user', 'admin'); await page.type('#password', 'admin'); const inputElement = await page.$('input[type=submit]'); await inputElement.click(); - await page.waitForNavigation({waitUntil: 'networkidle0'}); + await page.waitForNavigation({waitUntil: 'networkidle2'}); + await page.waitForSelector('#header'); + await page.$eval('body', function (e) { + // force relative timestamp to fixed value, since it breaks screenshot diffing + $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); + }); return await helper.delay(100); }, {viewport: resolution}); }) diff --git a/tests/ui-regression/test/publicSpec.js b/tests/ui-regression/test/publicSpec.js index 843f8f50cef..0893adf9a42 100644 --- a/tests/ui-regression/test/publicSpec.js +++ b/tests/ui-regression/test/publicSpec.js @@ -37,7 +37,7 @@ describe('public', function () { */ config.resolutions.forEach(function (resolution) { it('file-share-invalid.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/s/invalid', async function () { + return helper.takeAndCompare(this, 'index.php/s/invalid', async function () { }, {waitUntil: 'networkidle2', viewport: resolution}); }); }); @@ -48,7 +48,7 @@ describe('public', function () { var shareLink = {}; it('file-share-link', async function () { - return helper.takeAndCompare(this, '/index.php/apps/files', async function (page) { + return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) { const element = await page.$('[data-file="welcome.txt"] .action-share'); await element.click('[data-file="welcome.txt"] .action-share'); await page.waitForSelector('input.linkCheckbox'); @@ -71,7 +71,7 @@ describe('public', function () { config.resolutions.forEach(function (resolution) { it('file-share-valid.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/apps/files', async function (page) { + return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) { await page.goto(shareLink[page.url()]); await helper.delay(500); }, {waitUntil: 'networkidle2', viewport: resolution}); @@ -89,7 +89,7 @@ describe('public', function () { }); it('file-unshare', async function () { - return helper.takeAndCompare(this, '/index.php/apps/files', async function (page) { + return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) { const element = await page.$('[data-file="welcome.txt"] .action-share'); await element.click('[data-file="welcome.txt"] .action-share'); await page.waitForSelector('input.linkCheckbox'); diff --git a/tests/ui-regression/test/settingsSpec.js b/tests/ui-regression/test/settingsSpec.js index 560218c80f8..8b10b281fb6 100644 --- a/tests/ui-regression/test/settingsSpec.js +++ b/tests/ui-regression/test/settingsSpec.js @@ -33,24 +33,24 @@ describe('settings', function () { config.resolutions.forEach(function (resolution) { it('personal.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/settings/user', async function (page) { + return helper.takeAndCompare(this, 'index.php/settings/user', async function (page) { }, {viewport: resolution}); }); it('admin.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/settings/admin', async function (page) { + return helper.takeAndCompare(this, 'index.php/settings/admin', async function (page) { }, {viewport: resolution}); }); ['sharing', 'security', 'theming', 'encryption', 'additional', 'tips-tricks'].forEach(function(endpoint) { it('admin.' + endpoint + '.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/settings/admin/' + endpoint, async function (page) { + return helper.takeAndCompare(this, 'index.php/settings/admin/' + endpoint, async function (page) { }, {viewport: resolution, waitUntil: 'networkidle2'}); }); }); it('usermanagement.' + resolution.title, async function () { - return helper.takeAndCompare(this, '/index.php/settings/users', async function (page) { + return helper.takeAndCompare(this, 'index.php/settings/users', async function (page) { }, {viewport: resolution}); }); -- cgit v1.2.3 From d40f26b4c5ae80a28929d7bccc7ee92b3a961f3c Mon Sep 17 00:00:00 2001 From: Julius Härtl Date: Sun, 29 Jul 2018 16:40:42 +0200 Subject: Add custom network idle waiting and some JS injection to make tests more stable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- tests/ui-regression/helper.js | 62 +++++++++++++++++++++++++++++---- tests/ui-regression/test/filesSpec.js | 19 +++++----- tests/ui-regression/test/installSpec.js | 3 +- 3 files changed, 67 insertions(+), 17 deletions(-) (limited to 'tests/ui-regression/test/installSpec.js') diff --git a/tests/ui-regression/helper.js b/tests/ui-regression/helper.js index ed2a58ea70a..fd9a074f155 100644 --- a/tests/ui-regression/helper.js +++ b/tests/ui-regression/helper.js @@ -32,6 +32,8 @@ module.exports = { browser: null, pageBase: null, pageCompare: null, + lastBase: 0, + lastCompare: 0, init: async function (test) { this._outputDirectory = `${config.outputDirectory}/${test.title}`; if (!fs.existsSync(config.outputDirectory)) fs.mkdirSync(config.outputDirectory); @@ -54,6 +56,33 @@ module.exports = { this.pageCompare = await this.browser.newPage(); this.pageBase.setDefaultNavigationTimeout(60000); this.pageCompare.setDefaultNavigationTimeout(60000); + + const self = this; + this.pageCompare.on('requestfinished', function() { + self.lastCompare = Date.now(); + }); + this.pageBase.on('requestfinished', function() { + self.lastBase = Date.now(); + }); + }, + + awaitNetworkIdle: async function (seconds) { + var self = this; + return new Promise(function (resolve, reject) { + const timeout = setTimeout(function() { + reject(); + }, 10000) + const waitForFoo = function() { + const currentTime = Date.now() - seconds*1000; + if (self.lastBase < currentTime && self.lastCompare < currentTime) { + clearTimeout(timeout); + return resolve(); + } + setTimeout(waitForFoo, 100); + }; + waitForFoo(); + + }); }, login: async function (test) { @@ -71,8 +100,9 @@ module.exports = { await page.type('#user', 'admin'); await page.type('#password', 'admin'); const inputElement = await page.$('input[type=submit]'); - inputElement.click(); - return await page.waitForNavigation({waitUntil: 'networkidle0'}); + await inputElement.click(); + await page.waitForNavigation({waitUntil: 'networkidle2'}); + return await page.waitForSelector('#header'); }, takeAndCompare: async function (test, route, action, options) { @@ -107,16 +137,34 @@ module.exports = { this.pageCompare.goto(`${config.urlChange}${route}`, {waitUntil: options.waitUntil}) ]); } + + await this.pageBase.$eval('body', function (e) { + // force relative timestamp to fixed value, since it breaks screenshot diffing + $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); + }); + await this.pageCompare.$eval('body', function (e) { + // force relative timestamp to fixed value, since it breaks screenshot diffing + $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); + }); + var failed = null; try { - await Promise.all([ - action(this.pageBase), - action(this.pageCompare) - ]); + await this.pageBase.bringToFront(); + await action(this.pageBase); + await this.pageCompare.bringToFront(); + await action(this.pageCompare); } catch (err) { failed = err; } - await this.delay(100); + await this.awaitNetworkIdle(3); + await this.pageBase.$eval('body', function (e) { + $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); + $(':focus').blur(); + }); + await this.pageCompare.$eval('body', function (e) { + $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); + $(':focus').blur(); + }); await Promise.all([ this.pageBase.screenshot({ path: `${this._outputDirectory}/${fileName}.base.png`, diff --git a/tests/ui-regression/test/filesSpec.js b/tests/ui-regression/test/filesSpec.js index be507390f4e..7a029b2f311 100644 --- a/tests/ui-regression/test/filesSpec.js +++ b/tests/ui-regression/test/filesSpec.js @@ -41,43 +41,44 @@ describe('files', function () { await page.waitForSelector('.shareWithField'); await helper.delay(500); await page.$eval('body', e => { $('.shareWithField').blur() }); - }, {viewport: resolution, waitUntil: 'networkidle2'}); + }, {viewport: resolution}); }); it('file-popover.' + resolution.title, async function () { return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) { await page.click('[data-file=\'welcome.txt\'] .action-menu'); await page.waitForSelector('.fileActionsMenu'); - }, {viewport: resolution, waitUntil: 'networkidle2'}); + }, {viewport: resolution}); }); it('file-sidebar-details.' + resolution.title, async function() { return helper.takeAndCompare(this, undefined, async function (page) { await page.click('[data-file=\'welcome.txt\'] .fileActionsMenu [data-action=\'Details\']'); - await page.waitForSelector('#commentsTabView'); + await page.waitForSelector('[data-tabid=\'commentsTabView\']'); + await page.$eval('body', e => { $('.shareWithField').blur() }); await helper.delay(500); // wait for animation - }); + }, {viewport: resolution}); }); it('file-sidebar-details-sharing.' + resolution.title, async function() { return helper.takeAndCompare(this, undefined, async function (page) { let tab = await helper.childOfClassByText(page, 'tabHeaders', 'Sharing'); tab[0].click(); await page.waitForSelector('input.shareWithField'); - await helper.delay(500); // wait for animation await page.$eval('body', e => { $('.shareWithField').blur() }); - }); + await helper.delay(500); // wait for animation + }, {viewport: resolution}); }); it('file-sidebar-details-versions.' + resolution.title, async function() { return helper.takeAndCompare(this, undefined, async function (page) { let tab = await helper.childOfClassByText(page, 'tabHeaders', 'Versions'); tab[0].click(); await helper.delay(100); // wait for animation - }); + }, {viewport: resolution}); }); it('file-popover.favorite.' + resolution.title, async function () { return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) { await page.click('[data-file=\'welcome.txt\'] .action-menu'); await page.waitForSelector('.fileActionsMenu') await page.click('[data-file=\'welcome.txt\'] .fileActionsMenu [data-action=\'Favorite\']');; - }, {viewport: resolution, waitUntil: 'networkidle2'}); + }, {viewport: resolution}); }); it('file-favorites.' + resolution.title, async function () { @@ -90,7 +91,7 @@ describe('files', function () { } catch (err) {} await page.click('#app-navigation [data-id=\'favorites\'] a'); await helper.delay(500); // wait for animation - }, {viewport: resolution, waitUntil: 'networkidle2'}); + }, {viewport: resolution}); }); diff --git a/tests/ui-regression/test/installSpec.js b/tests/ui-regression/test/installSpec.js index bfb063bd467..d2107cdb61f 100644 --- a/tests/ui-regression/test/installSpec.js +++ b/tests/ui-regression/test/installSpec.js @@ -66,7 +66,8 @@ describe('install', function () { const password = await page.type('#adminpass', 'admin'); const inputElement = await page.$('input[type=submit]'); await inputElement.click(); - await page.waitForNavigation({waitUntil: 'networkidle0'}); + await page.waitForNavigation({waitUntil: 'networkidle2'}); + await page.waitForSelector('#header'); helper.pageBase.setDefaultNavigationTimeout(60000); helper.pageCompare.setDefaultNavigationTimeout(60000); }, { waitUntil: 'networkidle0', viewport: {w: 1920, h: 1080}}); -- cgit v1.2.3 From 5ef077840f5d0ce24e16404e7cbaf6d8fa25226f Mon Sep 17 00:00:00 2001 From: Julius Härtl Date: Mon, 30 Jul 2018 00:07:16 +0200 Subject: Improve test duration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl Reincrement network idle to 3 s Signed-off-by: Julius Härtl --- tests/ui-regression/config.js | 1 - tests/ui-regression/helper.js | 11 +---------- tests/ui-regression/test/installSpec.js | 8 ++++---- 3 files changed, 5 insertions(+), 15 deletions(-) (limited to 'tests/ui-regression/test/installSpec.js') diff --git a/tests/ui-regression/config.js b/tests/ui-regression/config.js index bb0f20240b8..c40efd722d7 100644 --- a/tests/ui-regression/config.js +++ b/tests/ui-regression/config.js @@ -31,7 +31,6 @@ module.exports = { {title: 'normal', w: 1024, h: 768}, {title: 'wide', w: 1920, h: 1080}, {title: 'qhd', w: 2560, h: 1440}, - {title: 'uhd', w: 3840, h: 2160}, ], /** diff --git a/tests/ui-regression/helper.js b/tests/ui-regression/helper.js index fd9a074f155..149c4059e99 100644 --- a/tests/ui-regression/helper.js +++ b/tests/ui-regression/helper.js @@ -138,15 +138,6 @@ module.exports = { ]); } - await this.pageBase.$eval('body', function (e) { - // force relative timestamp to fixed value, since it breaks screenshot diffing - $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); - }); - await this.pageCompare.$eval('body', function (e) { - // force relative timestamp to fixed value, since it breaks screenshot diffing - $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); - }); - var failed = null; try { await this.pageBase.bringToFront(); @@ -156,7 +147,6 @@ module.exports = { } catch (err) { failed = err; } - await this.awaitNetworkIdle(3); await this.pageBase.$eval('body', function (e) { $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); $(':focus').blur(); @@ -165,6 +155,7 @@ module.exports = { $('.live-relative-timestamp').removeClass('live-relative-timestamp').text('5 minutes ago'); $(':focus').blur(); }); + await this.awaitNetworkIdle(3); await Promise.all([ this.pageBase.screenshot({ path: `${this._outputDirectory}/${fileName}.base.png`, diff --git a/tests/ui-regression/test/installSpec.js b/tests/ui-regression/test/installSpec.js index d2107cdb61f..ffb4854f1b6 100644 --- a/tests/ui-regression/test/installSpec.js +++ b/tests/ui-regression/test/installSpec.js @@ -44,15 +44,15 @@ describe('install', function () { // (test, route, prepare, action, options return helper.takeAndCompare(this, undefined, async (page) => { await page.click('#showAdvanced'); - await helper.delay(500); - }); + await helper.delay(300); + }, { waitUntil: 'networkidle0', viewport: resolution}); }); it('show-advanced-mysql.' + resolution.title, async function () { // (test, route, prepare, action, options return helper.takeAndCompare(this, undefined, async (page) => { await page.click('label.mysql'); - await helper.delay(500); - }); + await helper.delay(300); + }, { waitUntil: 'networkidle0', viewport: resolution}); }); }); -- cgit v1.2.3