Merge pull request #18368 from owncloud/version-retention

Add versions obligation

4 files changed, 453 insertions, 7 deletions

diff --git a/apps/files_versions/appinfo/application.php b/apps/files_versions/appinfo/application.php
index bab36b48510..b61b03dab13 100644
--- a/apps/files_versions/appinfo/application.php
+++ b/apps/files_versions/appinfo/application.php
@@ -22,6 +22,7 @@
 namespace OCA\Files_Versions\AppInfo;
 
 use OCP\AppFramework\App;
+use OCA\Files_Versions\Expiration;
 
 class Application extends App {
 	public function __construct(array $urlParams = array()) {
@@ -33,5 +34,15 @@ class Application extends App {
 		 * Register capabilities
 		 */
 		$container->registerCapability('OCA\Files_Versions\Capabilities');
+
+		/*
+		 * Register expiration
+		 */
+		$container->registerService('Expiration', function($c) {
+			return  new Expiration(
+				$c->query('ServerContainer')->getConfig(),
+				$c->query('OCP\AppFramework\Utility\ITimeFactory')
+			);
+		});
 	}
 }
diff --git a/apps/files_versions/lib/expiration.php b/apps/files_versions/lib/expiration.php
new file mode 100644
index 00000000000..d42c62f0ee3
--- /dev/null
+++ b/apps/files_versions/lib/expiration.php
@@ -0,0 +1,185 @@
+<?php
+/**
+ * @author Victor Dubiniuk <dubiniuk@owncloud.com>
+ *
+ * @copyright Copyright (c) 2015, ownCloud, Inc.
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OCA\Files_Versions;
+
+use \OCP\IConfig;
+use \OCP\AppFramework\Utility\ITimeFactory;
+
+class Expiration {
+
+	// how long do we keep files a version if no other value is defined in the config file (unit: days)
+	const NO_OBLIGATION = -1;
+
+	/** @var ITimeFactory */
+	private $timeFactory;
+
+	/** @var string */
+	private $retentionObligation;
+
+	/** @var int */
+	private $minAge;
+
+	/** @var int */
+	private $maxAge;
+
+	/** @var bool */
+	private $canPurgeToSaveSpace;
+
+	public function __construct(IConfig $config,ITimeFactory $timeFactory){
+		$this->timeFactory = $timeFactory;
+		$this->retentionObligation = $config->getSystemValue('versions_retention_obligation', 'auto');
+
+		if ($this->retentionObligation !== 'disabled') {
+			$this->parseRetentionObligation();
+		}
+	}
+
+	/**
+	 * Is versions expiration enabled
+	 * @return bool
+	 */
+	public function isEnabled(){
+		return $this->retentionObligation !== 'disabled';
+	}
+
+	/**
+	 * Is default expiration active
+	 */
+	public function shouldAutoExpire(){
+		return $this->minAge === self::NO_OBLIGATION
+				|| $this->maxAge === self::NO_OBLIGATION;
+	}
+
+	/**
+	 * Check if given timestamp in expiration range
+	 * @param int $timestamp
+	 * @param bool $quotaExceeded
+	 * @return bool
+	 */
+	public function isExpired($timestamp, $quotaExceeded = false){
+		// No expiration if disabled
+		if (!$this->isEnabled()) {
+			return false;
+		}
+
+		// Purge to save space (if allowed)
+		if ($quotaExceeded && $this->canPurgeToSaveSpace) {
+			return true;
+		}
+
+		$time = $this->timeFactory->getTime();
+		// Never expire dates in future e.g. misconfiguration or negative time
+		// adjustment
+		if ($time<$timestamp) {
+			return false;
+		}
+
+		// Purge as too old
+		if ($this->maxAge !== self::NO_OBLIGATION) {
+			$maxTimestamp = $time - ($this->maxAge * 86400);
+			$isOlderThanMax = $timestamp < $maxTimestamp;
+		} else {
+			$isOlderThanMax = false;
+		}
+
+		if ($this->minAge !== self::NO_OBLIGATION) {
+			// older than Min obligation and we are running out of quota?
+			$minTimestamp = $time - ($this->minAge * 86400);
+			$isMinReached = ($timestamp < $minTimestamp) && $quotaExceeded;
+		} else {
+			$isMinReached = false;
+		}
+
+		return $isOlderThanMax || $isMinReached;
+	}
+
+	/**
+	* Read versions_retention_obligation, validate it 
+	* and set private members accordingly
+	*/
+	private function parseRetentionObligation(){
+		$splitValues = explode(',', $this->retentionObligation);
+		if (!isset($splitValues[0])) {
+			$minValue = 'auto';
+		} else {
+			$minValue = trim($splitValues[0]);
+		}
+
+		if (!isset($splitValues[1])) {
+			$maxValue = self::NO_OBLIGATION;
+		} else {
+			$maxValue = trim($splitValues[1]);
+		}
+
+		$isValid = true;
+		// Validate
+		if (!ctype_digit($minValue) && $minValue !== 'auto') {
+			$isValid = false;
+			\OC::$server->getLogger()->warning(
+					$minValue . ' is not a valid value for minimal versions retention obligation. Check versions_retention_obligation in your config.php. Falling back to auto.',
+					['app'=>'files_versions']
+			);
+		}
+
+		if (!ctype_digit($maxValue) && $maxValue !== 'auto') {
+			$isValid = false;
+			\OC::$server->getLogger()->warning(
+					$maxValue . ' is not a valid value for maximal versions retention obligation. Check versions_retention_obligation in your config.php. Falling back to auto.',
+					['app'=>'files_versions']
+			);
+		}
+
+		if (!$isValid){
+			$minValue = 'auto';
+			$maxValue = 'auto';
+		}
+
+
+		if ($minValue === 'auto' && $maxValue === 'auto') {
+			// Default: Delete anytime if space needed
+			$this->minAge = self::NO_OBLIGATION;
+			$this->maxAge = self::NO_OBLIGATION;
+			$this->canPurgeToSaveSpace = true;
+		} elseif ($minValue !== 'auto' && $maxValue === 'auto') {
+			// Keep for X days but delete anytime if space needed
+			$this->minAge = intval($minValue);
+			$this->maxAge = self::NO_OBLIGATION;
+			$this->canPurgeToSaveSpace = true;
+		} elseif ($minValue === 'auto' && $maxValue !== 'auto') {
+			// Delete anytime if space needed, Delete all older than max automatically
+			$this->minAge = self::NO_OBLIGATION;
+			$this->maxAge = intval($maxValue);
+			$this->canPurgeToSaveSpace = true;
+		} elseif ($minValue !== 'auto' && $maxValue !== 'auto') {
+			// Delete all older than max OR older than min if space needed
+
+			// Max < Min as per https://github.com/owncloud/core/issues/16301
+			if ($maxValue < $minValue) {
+				$maxValue = $minValue;
+			}
+
+			$this->minAge = intval($minValue);
+			$this->maxAge = intval($maxValue);
+			$this->canPurgeToSaveSpace = false;
+		}
+	}
+}
diff --git a/apps/files_versions/lib/storage.php b/apps/files_versions/lib/storage.php
index e0034f6165f..ba2b78ff4d2 100644
--- a/apps/files_versions/lib/storage.php
+++ b/apps/files_versions/lib/storage.php
@@ -40,6 +40,7 @@
 
 namespace OCA\Files_Versions;
 
+use OCA\Files_Versions\AppInfo\Application;
 use OCA\Files_Versions\Command\Expire;
 
 class Storage {
@@ -67,6 +68,9 @@ class Storage {
 		//until the end one version per week
 		6 => array('intervalEndsAfter' => -1,      'step' => 604800),
 	);
+	
+	/** @var \OCA\Files_Versions\AppInfo\Application */
+	private static $application;
 
 	public static function getUidAndFilename($filename) {
 		$uid = \OC\Files\Filesystem::getOwner($filename);
@@ -479,10 +483,36 @@ class Storage {
 	 * get list of files we want to expire
 	 * @param array $versions list of versions
 	 * @param integer $time
+	 * @param bool $quotaExceeded is versions storage limit reached
 	 * @return array containing the list of to deleted versions and the size of them
 	 */
-	protected static function getExpireList($time, $versions) {
+	protected static function getExpireList($time, $versions, $quotaExceeded = false) {
+		$expiration = self::getExpiration();
 
+		if ($expiration->shouldAutoExpire()) {
+			list($toDelete, $size) = self::getAutoExpireList($time, $versions);
+		} else {
+			$size = 0;
+			$toDelete = [];  // versions we want to delete
+		}
+
+		foreach ($versions as $key => $version) {
+			if ($expiration->isExpired($version['version'], $quotaExceeded) && !isset($toDelete[$key])) {
+				$size += $version['size'];
+				$toDelete[$key] = $version['path'] . '.v' . $version['version'];
+			}
+		}
+
+		return [$toDelete, $size];
+	}
+
+	/**
+	 * get list of files we want to expire
+	 * @param array $versions list of versions
+	 * @param integer $time
+	 * @return array containing the list of to deleted versions and the size of them
+	 */
+	protected static function getAutoExpireList($time, $versions) {
 		$size = 0;
 		$toDelete = array();  // versions we want to delete
 
@@ -529,7 +559,6 @@ class Storage {
 		}
 
 		return array($toDelete, $size);
-
 	}
 
 	/**
@@ -541,8 +570,12 @@ class Storage {
 	 * @param int $neededSpace requested versions size
 	 */
 	private static function scheduleExpire($uid, $fileName, $versionsSize = null, $neededSpace = 0) {
-		$command = new Expire($uid, $fileName, $versionsSize, $neededSpace);
-		\OC::$server->getCommandBus()->push($command);
+		// let the admin disable auto expire
+		$expiration = self::getExpiration();
+		if ($expiration->isEnabled()) {
+			$command = new Expire($uid, $fileName, $versionsSize, $neededSpace);
+			\OC::$server->getCommandBus()->push($command);
+		}
 	}
 
 	/**
@@ -555,7 +588,9 @@ class Storage {
 	 */
 	public static function expire($filename, $versionsSize = null, $offset = 0) {
 		$config = \OC::$server->getConfig();
-		if($config->getSystemValue('files_versions', Storage::DEFAULTENABLED)=='true') {
+		$expiration = self::getExpiration();
+		
+		if($config->getSystemValue('files_versions', Storage::DEFAULTENABLED)=='true' && $expiration->isEnabled()) {
 			list($uid, $filename) = self::getUidAndFilename($filename);
 			if (empty($filename)) {
 				// file maybe renamed or deleted
@@ -599,7 +634,7 @@ class Storage {
 			$allVersions = Storage::getVersions($uid, $filename);
 
 			$time = time();
-			list($toDelete, $sizeOfDeletedVersions) = self::getExpireList($time, $allVersions);
+			list($toDelete, $sizeOfDeletedVersions) = self::getExpireList($time, $allVersions, $availableSpace <= 0);
 
 			$availableSpace = $availableSpace + $sizeOfDeletedVersions;
 			$versionsSize = $versionsSize - $sizeOfDeletedVersions;
@@ -610,7 +645,7 @@ class Storage {
 				$allVersions = $result['all'];
 
 				foreach ($result['by_file'] as $versions) {
-					list($toDeleteNew, $size) = self::getExpireList($time, $versions);
+					list($toDeleteNew, $size) = self::getExpireList($time, $versions, $availableSpace <= 0);
 					$toDelete = array_merge($toDelete, $toDeleteNew);
 					$sizeOfDeletedVersions += $size;
 				}
@@ -672,4 +707,15 @@ class Storage {
 		}
 	}
 
+	/**
+	 * Static workaround
+	 * @return Expiration
+	 */
+	protected static function getExpiration(){
+		if (is_null(self::$application)) {
+			self::$application = new Application();
+		}
+		return self::$application->getContainer()->query('Expiration');
+	}
+
 }
diff --git a/apps/files_versions/tests/expirationtest.php b/apps/files_versions/tests/expirationtest.php
new file mode 100644
index 00000000000..54024b85b78
--- /dev/null
+++ b/apps/files_versions/tests/expirationtest.php
@@ -0,0 +1,204 @@
+<?php
+/**
+ * @author Victor Dubiniuk <dubiniuk@owncloud.com>
+ *
+ * @copyright Copyright (c) 2015, ownCloud, Inc.
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OCA\Files_Versions\Tests;
+
+use \OCA\Files_Versions\Expiration;
+
+class Expiration_Test extends \Test\TestCase {
+	const SECONDS_PER_DAY = 86400; //60*60*24
+
+	public function expirationData(){
+		$today = 100*self::SECONDS_PER_DAY;
+		$back10Days = (100-10)*self::SECONDS_PER_DAY;
+		$back20Days = (100-20)*self::SECONDS_PER_DAY;
+		$back30Days = (100-30)*self::SECONDS_PER_DAY;
+		$back35Days = (100-35)*self::SECONDS_PER_DAY;
+
+		// it should never happen, but who knows :/
+		$ahead100Days = (100+100)*self::SECONDS_PER_DAY;
+
+		return [
+			// Expiration is disabled - always should return false
+			[ 'disabled', $today, $back10Days, false, false],
+			[ 'disabled', $today, $back10Days, true, false],
+			[ 'disabled', $today, $ahead100Days, true, false],
+
+			// Default: expire in 30 days or earlier when quota requirements are met
+			[ 'auto', $today, $back10Days, false, false],
+			[ 'auto', $today, $back35Days, false, false],
+			[ 'auto', $today, $back10Days, true, true],
+			[ 'auto', $today, $back35Days, true, true],
+			[ 'auto', $today, $ahead100Days, true, true],
+
+			// The same with 'auto'
+			[ 'auto, auto', $today, $back10Days, false, false],
+			[ 'auto, auto', $today, $back35Days, false, false],
+			[ 'auto, auto', $today, $back10Days, true, true],
+			[ 'auto, auto', $today, $back35Days, true, true],
+
+			// Keep for 15 days but expire anytime if space needed
+			[ '15, auto', $today, $back10Days, false, false],
+			[ '15, auto', $today, $back20Days, false, false],
+			[ '15, auto', $today, $back10Days, true, true],
+			[ '15, auto', $today, $back20Days, true, true],
+			[ '15, auto', $today, $ahead100Days, true, true],
+
+			// Expire anytime if space needed, Expire all older than max
+			[ 'auto, 15', $today, $back10Days, false, false],
+			[ 'auto, 15', $today, $back20Days, false, true],
+			[ 'auto, 15', $today, $back10Days, true, true],
+			[ 'auto, 15', $today, $back20Days, true, true],
+			[ 'auto, 15', $today, $ahead100Days, true, true],
+
+			// Expire all older than max OR older than min if space needed
+			[ '15, 25', $today, $back10Days, false, false],
+			[ '15, 25', $today, $back20Days, false, false],
+			[ '15, 25', $today, $back30Days, false, true],
+			[ '15, 25', $today, $back10Days, false, false],
+			[ '15, 25', $today, $back20Days, true, true],
+			[ '15, 25', $today, $back30Days, true, true],
+			[ '15, 25', $today, $ahead100Days, true, false],
+
+			// Expire all older than max OR older than min if space needed
+			// Max<Min case
+			[ '25, 15', $today, $back10Days, false, false],
+			[ '25, 15', $today, $back20Days, false, false],
+			[ '25, 15', $today, $back30Days, false, true],
+			[ '25, 15', $today, $back10Days, false, false],
+			[ '25, 15', $today, $back20Days, true, false],
+			[ '25, 15', $today, $back30Days, true, true],
+			[ '25, 15', $today, $ahead100Days, true, false],
+		];
+	}
+
+	/**
+	 * @dataProvider expirationData
+	 *
+	 * @param string $retentionObligation
+	 * @param int $timeNow
+	 * @param int $timestamp
+	 * @param bool $quotaExceeded
+	 * @param string $expectedResult
+	 */
+	public function testExpiration($retentionObligation, $timeNow, $timestamp, $quotaExceeded, $expectedResult){
+		$mockedConfig = $this->getMockedConfig($retentionObligation);
+		$mockedTimeFactory = $this->getMockedTimeFactory($timeNow);
+
+		$expiration = new Expiration($mockedConfig, $mockedTimeFactory);
+		$actualResult = $expiration->isExpired($timestamp, $quotaExceeded);
+
+		$this->assertEquals($expectedResult, $actualResult);
+	}
+
+
+	public function configData(){
+		return [
+			[ 'disabled', null, null, null],
+			[ 'auto', Expiration::NO_OBLIGATION, Expiration::NO_OBLIGATION, true ],
+			[ 'auto,auto', Expiration::NO_OBLIGATION, Expiration::NO_OBLIGATION, true ],
+			[ 'auto, auto', Expiration::NO_OBLIGATION, Expiration::NO_OBLIGATION, true ],
+			[ 'auto, 3', Expiration::NO_OBLIGATION, 3, true ],
+			[ '5, auto', 5, Expiration::NO_OBLIGATION, true ],
+			[ '3, 5', 3, 5, false ],
+			[ '10, 3', 10, 10, false ],
+			[ 'g,a,r,b,a,g,e',  Expiration::NO_OBLIGATION, Expiration::NO_OBLIGATION, true ],
+			[ '-3,8',  Expiration::NO_OBLIGATION, Expiration::NO_OBLIGATION, true ]
+		];
+	}
+
+
+	/**
+	 * @dataProvider configData
+	 *
+	 * @param string $configValue
+	 * @param int $expectedMinAge
+	 * @param int $expectedMaxAge
+	 * @param bool $expectedCanPurgeToSaveSpace
+	 */
+	public function testParseRetentionObligation($configValue, $expectedMinAge, $expectedMaxAge, $expectedCanPurgeToSaveSpace){
+		$mockedConfig = $this->getMockedConfig($configValue);
+		$mockedTimeFactory = $this->getMockedTimeFactory(
+				time()
+		);
+
+		$expiration = new Expiration($mockedConfig, $mockedTimeFactory);
+		$this->assertAttributeEquals($expectedMinAge, 'minAge', $expiration);
+		$this->assertAttributeEquals($expectedMaxAge, 'maxAge', $expiration);
+		$this->assertAttributeEquals($expectedCanPurgeToSaveSpace, 'canPurgeToSaveSpace', $expiration);
+	}
+
+	/**
+	 *
+	 * @param int $time
+	 * @return \OCP\AppFramework\Utility\ITimeFactory
+	 */
+	private function getMockedTimeFactory($time){
+		$mockedTimeFactory = $this->getMockBuilder('\OCP\AppFramework\Utility\ITimeFactory')
+				->disableOriginalConstructor()
+				->setMethods(['getTime'])
+				->getMock()
+		;
+		$mockedTimeFactory->expects($this->any())->method('getTime')->will(
+				$this->returnValue($time)
+		);
+
+		return $mockedTimeFactory;
+	}
+
+	/**
+	 *
+	 * @param string $returnValue
+	 * @return \OCP\IConfig
+	 */
+	private function getMockedConfig($returnValue){
+		$mockedConfig = $this->getMockBuilder('\OCP\IConfig')
+				->disableOriginalConstructor()
+				->setMethods(
+					[
+						'setSystemValues',
+						'setSystemValue',
+						'getSystemValue',
+						'deleteSystemValue',
+						'getAppKeys',
+						'setAppValue',
+						'getAppValue',
+						'deleteAppValue',
+						'deleteAppValues',
+						'setUserValue',
+						'getUserValue',
+						'getUserValueForUsers',
+						'getUserKeys',
+						'deleteUserValue',
+						'deleteAllUserValues',
+						'deleteAppFromAllUsers',
+						'getUsersForUserValue'
+					]
+				)
+				->getMock()
+		;
+		$mockedConfig->expects($this->any())->method('getSystemValue')->will(
+				$this->returnValue($returnValue)
+		);
+
+		return $mockedConfig;
+	}
+}