Port of LDAP Wizard: get correct total no of users, groups and complete list of groups on big setups #9002

fix PHPdoc

Conflicts:
	apps/user_ldap/lib/connection.php

add method to count groups on LDAP

Conflicts:
	apps/user_ldap/lib/access.php

LDAP Wizard: count users and groups with the power of paged search

Conflicts:
	apps/user_ldap/lib/wizard.php

consolidate requirement check

fix PHPdoc

Conflicts:
	apps/user_ldap/lib/access.php

Wizard: get really all groups from LDAP by power of Paged Search

Conflicts:
	apps/user_ldap/lib/wizard.php

make all this work in an early configuration state in the wizard by marking  the config active and ignoring the validation state.

Conflicts:
	apps/user_ldap/lib/connection.php

simplify two methods a bit, because they are not used for group search anymore

Conflicts:
	apps/user_ldap/lib/wizard.php

remove unused vars; increase scrutinizer happiness

4 files changed, 130 insertions, 70 deletions

diff --git a/apps/user_ldap/lib/access.php b/apps/user_ldap/lib/access.php
index d289627bead..9eb5836399e 100644
--- a/apps/user_ldap/lib/access.php
+++ b/apps/user_ldap/lib/access.php
@@ -724,10 +724,18 @@ class Access extends LDAPUtility implements user\IUserTools {
 	}
 
 	/**
-	 * prepares and executes an LDAP search operation
-	 * @param string $filter the LDAP filter for the search
-	 * @param array $base an array containing the LDAP subtree(s) that shall be searched
-	 * @param string|string[] $attr optional, array, one or more attributes that shall be
+	 * returns the number of available groups
+	 * @param string $filter the LDAP search filter
+	 * @param string[] $attr optional
+	 * @param int|null $limit
+	 * @param int|null $offset
+	 * @return int|bool
+	 */
+	public function countGroups($filter, $attr = array('dn'), $limit = null, $offset = null) {
+		return $this->count($filter, $this->connection->ldapBaseGroups, $attr, $limit, $offset);
+	}
+
+	/**
 	 * retrieved. Results will according to the order in the array.
 	 * @param int $limit optional, maximum results to be counted
 	 * @param int $offset optional, a starting point
@@ -1026,9 +1034,10 @@ class Access extends LDAPUtility implements user\IUserTools {
 	}
 
 	/**
-	 * combines the input filters with AND
+	 * combines the input filters with OR
 	 * @param string[] $filters the filters to connect
 	 * @return string the combined filter
+	 * Combines Filter arguments with OR
 	 */
 	public function combineFilterWithOr($filters) {
 		return $this->combineFilter($filters, '|');
diff --git a/apps/user_ldap/lib/connection.php b/apps/user_ldap/lib/connection.php
index 52f6c5ceb10..bafb0e0b895 100644
--- a/apps/user_ldap/lib/connection.php
+++ b/apps/user_ldap/lib/connection.php
@@ -41,11 +41,13 @@ class Connection extends LDAPUtility {
 
 	protected $doNotValidate = false;
 
+	protected $ignoreValidation = false;
+
 	/**
 	 * Constructor
 	 * @param ILDAPWrapper $ldap
 	 * @param string $configPrefix a string with the prefix for the configkey column (appconfig table)
-	 * @param string $configID a string with the value for the appid column (appconfig table) or null for on-the-fly connections
+	 * @param string|null $configID a string with the value for the appid column (appconfig table) or null for on-the-fly connections
 	 */
 	public function __construct(ILDAPWrapper $ldap, $configPrefix = '', $configID = 'user_ldap') {
 		parent::__construct($ldap);
@@ -117,6 +119,16 @@ class Connection extends LDAPUtility {
 	}
 
 	/**
+	 * sets whether the result of the configuration validation shall
+	 * be ignored when establishing the connection. Used by the Wizard
+	 * in early configuration state.
+	 * @param bool $state
+	 */
+	public function setIgnoreValidation($state) {
+		$this->ignoreValidation = (bool)$state;
+	}
+
+	/**
 	 * initializes the LDAP backend
 	 * @param bool $force read the config settings no matter what
 	 */
@@ -466,7 +478,7 @@ class Connection extends LDAPUtility {
 		if(!$phpLDAPinstalled) {
 			return false;
 		}
-		if(!$this->configured) {
+		if(!$this->ignoreValidation && !$this->configured) {
 			\OCP\Util::writeLog('user_ldap',
 								'Configuration is invalid, cannot connect',
 								\OCP\Util::WARN);
diff --git a/apps/user_ldap/lib/wizard.php b/apps/user_ldap/lib/wizard.php
index b8a0e5ad799..1c53a492644 100644
--- a/apps/user_ldap/lib/wizard.php
+++ b/apps/user_ldap/lib/wizard.php
@@ -64,39 +64,53 @@ class Wizard extends LDAPUtility {
 	}
 
 	/**
-	 * @return WizardResult
-	 * @throws \Exception
+	 * counts entries in the LDAP directory
+	 * @param string $filter the LDAP search filter
+	 * @param string $type a string being either 'users' or 'groups';
+	 * @return int|bool
 	 */
-	public function countGroups() {
-		if(!$this->checkRequirements(array('ldapHost',
-										   'ldapPort',
-										   'ldapBase',
-										   ))) {
-			return  false;
+	public function countEntries($filter, $type) {
+		$reqs = array('ldapHost', 'ldapPort', 'ldapBase');
+		if($type === 'users') {
+			$reqs[] = 'ldapUserFilter';
+		}
+		if(!$this->checkRequirements($reqs)) {
+			throw new \Exception('Requirements not met', 400);
 		}
 
-		$base = $this->configuration->ldapBase[0];
+		$ldapAccess = $this->getAccess();
+		if($type === 'groups') {
+			$result =  $ldapAccess->countGroups($filter);
+		} else if($type === 'users') {
+			$result = $ldapAccess->countUsers($filter);
+		} else {
+			throw new \Exception('internal error: invald object type', 500);
+		}
+
+		return $result;
+	}
+
+	public function countGroups() {
 		$filter = $this->configuration->ldapGroupFilter;
-		\OCP\Util::writeLog('user_ldap', 'Wiz: g filter '. print_r($filter, true), \OCP\Util::DEBUG);
-		$l = \OC_L10N::get('user_ldap');
+
 		if(empty($filter)) {
-			$output = $l->n('%s group found', '%s groups found', 0, array(0));
+			$output = self::$l->n('%s group found', '%s groups found', 0, array(0));
 			$this->result->addChange('ldap_group_count', $output);
 			return $this->result;
 		}
-		$cr = $this->getConnection();
-		if(!$cr) {
-			throw new \Exception('Could not connect to LDAP');
-		}
-		$rr = $this->ldap->search($cr, $base, $filter, array('dn'));
-		if(!$this->ldap->isResource($rr)) {
+
+		try {
+			$groupsTotal = $this->countEntries($filter, 'groups');
+		} catch (\Exception $e) {
+			//400 can be ignored, 500 is forwarded
+			if($e->getCode() === 500) {
+				throw $e;
+			}
 			return false;
 		}
-		$entries = $this->ldap->countEntries($cr, $rr);
-		$entries = ($entries !== false) ? $entries : 0;
-		$output = $l->n('%s group found', '%s groups found', $entries, $entries);
+		$groupsTotal = ($groupsTotal !== false) ? $groupsTotal : 0;
+		$output = self::$l->n('%s group found', '%s groups found', $groupsTotal, $groupsTotal);
 		$this->result->addChange('ldap_group_count', $output);
-
 		return $this->result;
 	}
 
@@ -105,31 +119,12 @@ class Wizard extends LDAPUtility {
 	 * @throws \Exception
 	 */
 	public function countUsers() {
-		if(!$this->checkRequirements(array('ldapHost',
-										   'ldapPort',
-										   'ldapBase',
-										   'ldapUserFilter',
-										   ))) {
-			return  false;
-		}
-
-		$cr = $this->getConnection();
-		if(!$cr) {
-			throw new \Exception('Could not connect to LDAP');
-		}
-
-		$base = $this->configuration->ldapBase[0];
 		$filter = $this->configuration->ldapUserFilter;
-		$rr = $this->ldap->search($cr, $base, $filter, array('dn'));
-		if(!$this->ldap->isResource($rr)) {
-			return false;
-		}
-		$entries = $this->ldap->countEntries($cr, $rr);
-		$entries = ($entries !== false) ? $entries : 0;
-		$l = \OC_L10N::get('user_ldap');
-		$output = $l->n('%s user found', '%s users found', $entries, $entries);
-		$this->result->addChange('ldap_user_count', $output);
 
+		$usersTotal = $this->countEntries($filter, 'users');
+		$usersTotal = ($usersTotal !== false) ? $usersTotal : 0;
+		$output = self::$l->n('%s user found', '%s users found', $usersTotal, $usersTotal);
+		$this->result->addChange('ldap_user_count', $output);
 		return $this->result;
 	}
 
@@ -273,8 +268,7 @@ class Wizard extends LDAPUtility {
 			throw new \Exception('Could not connect to LDAP');
 		}
 
-		$obClasses = array('posixGroup', 'group', 'zimbraDistributionList', '*');
-		$this->determineFeature($obClasses, 'cn', $dbKey, $confKey);
+		$this->fetchGroups($dbKey, $confKey);
 
 		if($testMemberOf) {
 			$this->configuration->hasMemberOfFilterSupport = $this->testMemberOf();
@@ -288,9 +282,48 @@ class Wizard extends LDAPUtility {
 	}
 
 	/**
-	 * @return bool|WizardResult
-	 * @throws \Exception
+	 * fetches all groups from LDAP
+	 * @param string $dbKey
+	 * @param string $confKey
 	 */
+	public function fetchGroups($dbKey, $confKey) {
+		$obclasses = array('posixGroup', 'group', 'zimbraDistributionList', 'groupOfNames');
+		$ldapAccess = $this->getAccess();
+
+		$filterParts = array();
+		foreach($obclasses as $obclass) {
+			$filterParts[] = 'objectclass='.$obclass;
+		}
+		//we filter for everything
+		//- that looks like a group and
+		//- has the group display name set
+		$filter = $ldapAccess->combineFilterWithOr($filterParts);
+		$filter = $ldapAccess->combineFilterWithAnd(array($filter, 'cn=*'));
+
+		$limit = 400;
+		$offset = 0;
+		do {
+			$result = $ldapAccess->searchGroups($filter, array('cn'), $limit, $offset);
+			foreach($result as $item) {
+				$groups[] = $item[0];
+			}
+			$offset += $limit;
+		} while (count($groups) > 0 && count($groups) % $limit === 0);
+
+		if(count($groups) > 0) {
+			natsort($groups);
+			$this->result->addOptions($dbKey, array_values($groups));
+		} else {
+			throw new \Exception(self::$l->t('Could not find the desired feature'));
+		}
+
+		$setFeatures = $this->configuration->$confKey;
+		if(is_array($setFeatures) && !empty($setFeatures)) {
+			//something is already configured? pre-select it.
+			$this->result->addChange($dbKey, $setFeatures);
+		}
+	}
+
 	public function determineGroupMemberAssoc() {
 		if(!$this->checkRequirements(array('ldapHost',
 										   'ldapPort',
@@ -909,15 +942,13 @@ class Wizard extends LDAPUtility {
 	 * specified attribute
 	 * @param string[] $filters array, the filters that shall be used in the search
 	 * @param string $attr the attribute of which a list of values shall be returned
-	 * @param bool $lfw whether the last filter is a wildcard which shall not
-	 * be processed if there were already findings, defaults to true
 	 * @param int $dnReadLimit the amount of how many DNs should be analyzed.
 	 * The lower, the faster
 	 * @param string $maxF string. if not null, this variable will have the filter that
 	 * yields most result entries
 	 * @return array|false an array with the values on success, false otherwise
 	 */
-	public function cumulativeSearchOnAttribute($filters, $attr, $lfw = true, $dnReadLimit = 3, &$maxF = null) {
+	public function cumulativeSearchOnAttribute($filters, $attr, $dnReadLimit = 3, &$maxF = null) {
 		$dnRead = array();
 		$foundItems = array();
 		$maxEntries = 0;
@@ -935,7 +966,7 @@ class Wizard extends LDAPUtility {
 			$lastFilter = $filters[count($filters)-1];
 		}
 		foreach($filters as $filter) {
-			if($lfw && $lastFilter === $filter && count($foundItems) > 0) {
+			if($lastFilter === $filter && count($foundItems) > 0) {
 				//skip when the filter is a wildcard and results were found
 				continue;
 			}
@@ -1005,16 +1036,11 @@ class Wizard extends LDAPUtility {
 
 		//how deep to dig?
 		//When looking for objectclasses, testing few entries is sufficient,
-		//when looking for group we need to get all names, though.
-		if(strtolower($attr) === 'objectclass') {
-			$dig = 3;
-		} else {
-			$dig = 0;
-		}
+		$dig = 3;
 
 		$availableFeatures =
 			$this->cumulativeSearchOnAttribute($objectclasses, $attr,
-											   true, $dig, $maxEntryObjC);
+											   $dig, $maxEntryObjC);
 		if(is_array($availableFeatures)
 		   && count($availableFeatures) > 0) {
 			natcasesort($availableFeatures);
@@ -1073,6 +1099,19 @@ class Wizard extends LDAPUtility {
 	}
 
 	/**
+	 * creates and returns an Access instance
+	 * @return \OCA\user_ldap\lib\Access
+	 */
+	private function getAccess() {
+		$con = new Connection($this->ldap, '', null);
+		$con->setConfiguration($this->configuration->getConfiguration());
+		$con->ldapConfigurationActive = true;
+		$con->setIgnoreValidation(true);
+		$ldapAccess = new Access($con, $this->ldap);
+		return $ldapAccess;
+	}
+
+	/**
 	 * @return bool|mixed
 	 */
 	private function getConnection() {
diff --git a/apps/user_ldap/tests/wizard.php b/apps/user_ldap/tests/wizard.php
index 2b5cabc705d..ff5ee010b71 100644
--- a/apps/user_ldap/tests/wizard.php
+++ b/apps/user_ldap/tests/wizard.php
@@ -127,7 +127,7 @@ class Test_Wizard extends \PHPUnit_Framework_TestCase {
 
 		# The following expectations are the real test #
 		$filters = array('f1', 'f2', '*');
-		$wizard->cumulativeSearchOnAttribute($filters, 'cn', true, 5);
+		$wizard->cumulativeSearchOnAttribute($filters, 'cn', 5);
 		unset($uidnumber);
 	}
 
@@ -203,8 +203,8 @@ class Test_Wizard extends \PHPUnit_Framework_TestCase {
 
 		# The following expectations are the real test #
 		$filters = array('f1', 'f2', '*');
-		$wizard->cumulativeSearchOnAttribute($filters, 'cn', true, 0);
+		$wizard->cumulativeSearchOnAttribute($filters, 'cn', 0);
 		unset($uidnumber);
 	}
 
-}
\ No newline at end of file
+}