Merge pull request #6221 from leo-b/ldap_get_all_groups

fix updateGroups background job for ldap servers with > 1000 groups

apps/user_ldap/group_ldap.php

@@ -299,9 +299,9 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface {
 	 * @brief get a list of all groups
 	 * @returns array with group names
 	 *
-	 * Returns a list with all groups
+	 * Returns a list with all groups (used by getGroups)
 	 */
-	public function getGroups($search = '', $limit = -1, $offset = 0) {
+	protected function getGroupsChunk($search = '', $limit = -1, $offset = 0) {
 		if(!$this->enabled) {
 			return array();
 		}
@@ -334,6 +334,48 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface {
 		return $ldap_groups;
 	}
 
+	/**
+	 * @brief get a list of all groups using a paged search
+	 * @returns array with group names
+	 *
+	 * Returns a list with all groups
+   	 * Uses a paged search if available to override a
+   	 * server side search limit.
+   	 * (active directory has a limit of 1000 by default)
+	 */
+	public function getGroups($search = '', $limit = -1, $offset = 0) {
+		if(!$this->enabled) {
+			return array();
+		}
+		$pagingsize = $this->access->connection->ldapPagingSize;
+		if ((! $this->access->connection->hasPagedResultSupport)
+		   	|| empty($pagingsize)) {
+			return $this->getGroupsChunk($search, $limit, $offset);
+		}
+		$maxGroups = 100000; // limit max results (just for safety reasons)
+		if ($limit > -1) {
+		   $overallLimit = min($limit, $maxGroups);
+		} else {
+		   $overallLimit = $maxGroups;
+		}
+		$chunkOffset = $offset;
+		$allGroups = array();
+		while ($chunkOffset < $overallLimit) {
+			$chunkLimit = min($pagingsize, $overallLimit - $chunkOffset);
+			$ldapGroups = $this->getGroupsChunk($search, $chunkLimit, $chunkOffset);
+			$nread = count($ldapGroups);
+			\OCP\Util::writeLog('user_ldap', 'getGroups('.$search.'): read '.$nread.' at offset '.$chunkOffset.' (limit: '.$chunkLimit.')', \OCP\Util::DEBUG);
+			if ($nread) {
+				$allGroups = array_merge($allGroups, $ldapGroups);
+				$chunkOffset += $nread;
+			}
+			if ($nread < $chunkLimit) {
+				break;
+			}
+		}
+		return $allGroups;
+	}
+
 	public function groupMatchesFilter($group) {
 		return (strripos($group, $this->groupSearch) !== false);
 	}

apps/user_ldap/lib/configuration.php

@@ -77,6 +77,7 @@ class Configuration {
 		'ldapExpertUUIDGroupAttr' => null,
 		'lastJpegPhotoLookup' => null,
 		'ldapNestedGroups' => false,
+		'ldapPagingSize' => null,
 	);
 
 	/**
@@ -344,6 +345,7 @@ class Configuration {
 			'has_memberof_filter_support'		=> 0,
 			'last_jpegPhoto_lookup'				=> 0,
 			'ldap_nested_groups'		        => 0,
+			'ldap_paging_size'					=> 500,
 		);
 	}
 
@@ -395,7 +397,8 @@ class Configuration {
 			'ldap_expert_uuid_group_attr'		=> 'ldapExpertUUIDGroupAttr',
 			'has_memberof_filter_support'		=> 'hasMemberOfFilterSupport',
 			'last_jpegPhoto_lookup'				=> 'lastJpegPhotoLookup',
-			'ldap_nested_groups'                    => 'ldapNestedGroups',
+			'ldap_nested_groups'                => 'ldapNestedGroups',
+			'ldap_paging_size'                  => 'ldapPagingSize',
 		);
 		return $array;
 	}

apps/user_ldap/templates/settings.php

@@ -37,6 +37,7 @@
 				<p><label for="ldap_attributes_for_group_search"><?php p($l->t('Group Search Attributes'));?></label><textarea id="ldap_attributes_for_group_search" name="ldap_attributes_for_group_search" placeholder="<?php p($l->t('Optional; one attribute per line'));?>" data-default="<?php p($_['ldap_attributes_for_group_search_default']); ?>" title="<?php p($l->t('Group Search Attributes'));?>"></textarea></p>
 				<p><label for="ldap_group_member_assoc_attribute"><?php p($l->t('Group-Member association'));?></label><select id="ldap_group_member_assoc_attribute" name="ldap_group_member_assoc_attribute" data-default="<?php p($_['ldap_group_member_assoc_attribute_default']); ?>" ><option value="uniqueMember"<?php if (isset($_['ldap_group_member_assoc_attribute']) && ($_['ldap_group_member_assoc_attribute'] === 'uniqueMember')) p(' selected'); ?>>uniqueMember</option><option value="memberUid"<?php if (isset($_['ldap_group_member_assoc_attribute']) && ($_['ldap_group_member_assoc_attribute'] === 'memberUid')) p(' selected'); ?>>memberUid</option><option value="member"<?php if (isset($_['ldap_group_member_assoc_attribute']) && ($_['ldap_group_member_assoc_attribute'] === 'member')) p(' selected'); ?>>member (AD)</option></select></p>
 				<p><label for="ldap_nested_groups"><?php p($l->t('Nested Groups'));?></label><input type="checkbox" id="ldap_nested_groups" name="ldap_nested_groups" value="1" data-default="<?php p($_['ldap_nested_groups_default']); ?>"  title="<?php p($l->t('When switched on, groups that contain groups are supported. (Only works if the group member attribute contains DNs.)'));?>" /></p>
+				<p><label for="ldap_paging_size"><?php p($l->t('Paging chunksize'));?></label><input type="number" id="ldap_paging_size" name="ldap_paging_size" title="<?php p($l->t('Chunksize used for paged LDAP searches that may return bulky results like user or group enumeration. (Setting it 0 disables paged LDAP searches in those situations.)'));?>" data-default="<?php p($_['ldap_paging_size']); ?>" /></p>
 			</div>
 			<h3><?php p($l->t('Special Attributes'));?></h3>
 			<div>