diff options
Diffstat (limited to 'apps/files_versions/lib/Storage.php')
| -rw-r--r-- | apps/files_versions/lib/Storage.php | 1007 |
1 files changed, 1007 insertions, 0 deletions
diff --git a/apps/files_versions/lib/Storage.php b/apps/files_versions/lib/Storage.php new file mode 100644 index 00000000000..6d53a19a518 --- /dev/null +++ b/apps/files_versions/lib/Storage.php @@ -0,0 +1,1007 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OCA\Files_Versions; + +use OC\Files\Filesystem; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchQuery; +use OC\Files\View; +use OC\User\NoUserException; +use OC_User; +use OCA\Files_Sharing\SharedMount; +use OCA\Files_Versions\AppInfo\Application; +use OCA\Files_Versions\Command\Expire; +use OCA\Files_Versions\Db\VersionsMapper; +use OCA\Files_Versions\Events\CreateVersionEvent; +use OCA\Files_Versions\Versions\IVersionManager; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\Command\IBus; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files; +use OCP\Files\FileInfo; +use OCP\Files\Folder; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IRootFolder; +use OCP\Files\Node; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Storage\IWriteStreamStorage; +use OCP\Files\StorageInvalidException; +use OCP\Files\StorageNotAvailableException; +use OCP\IURLGenerator; +use OCP\IUser; +use OCP\IUserManager; +use OCP\Lock\ILockingProvider; +use OCP\Server; +use OCP\Util; +use Psr\Log\LoggerInterface; + +class Storage { + public const DEFAULTENABLED = true; + public const DEFAULTMAXSIZE = 50; // unit: percentage; 50% of available disk space/quota + public const VERSIONS_ROOT = 'files_versions/'; + + public const DELETE_TRIGGER_MASTER_REMOVED = 0; + public const DELETE_TRIGGER_RETENTION_CONSTRAINT = 1; + public const DELETE_TRIGGER_QUOTA_EXCEEDED = 2; + + // files for which we can remove the versions after the delete operation was successful + private static $deletedFiles = []; + + private static $sourcePathAndUser = []; + + private static $max_versions_per_interval = [ + //first 10sec, one version every 2sec + 1 => ['intervalEndsAfter' => 10, 'step' => 2], + //next minute, one version every 10sec + 2 => ['intervalEndsAfter' => 60, 'step' => 10], + //next hour, one version every minute + 3 => ['intervalEndsAfter' => 3600, 'step' => 60], + //next 24h, one version every hour + 4 => ['intervalEndsAfter' => 86400, 'step' => 3600], + //next 30days, one version per day + 5 => ['intervalEndsAfter' => 2592000, 'step' => 86400], + //until the end one version per week + 6 => ['intervalEndsAfter' => -1, 'step' => 604800], + ]; + + /** @var Application */ + private static $application; + + /** + * get the UID of the owner of the file and the path to the file relative to + * owners files folder + * + * @param string $filename + * @return array + * @throws NoUserException + */ + public static function getUidAndFilename($filename) { + $uid = Filesystem::getOwner($filename); + $userManager = Server::get(IUserManager::class); + // if the user with the UID doesn't exists, e.g. because the UID points + // to a remote user with a federated cloud ID we use the current logged-in + // user. We need a valid local user to create the versions + if (!$userManager->userExists($uid)) { + $uid = OC_User::getUser(); + } + Filesystem::initMountPoints($uid); + if ($uid !== OC_User::getUser()) { + $info = Filesystem::getFileInfo($filename); + $ownerView = new View('/' . $uid . '/files'); + try { + $filename = $ownerView->getPath($info['fileid']); + // make sure that the file name doesn't end with a trailing slash + // can for example happen single files shared across servers + $filename = rtrim($filename, '/'); + } catch (NotFoundException $e) { + $filename = null; + } + } + return [$uid, $filename]; + } + + /** + * Remember the owner and the owner path of the source file + * + * @param string $source source path + */ + public static function setSourcePathAndUser($source) { + [$uid, $path] = self::getUidAndFilename($source); + self::$sourcePathAndUser[$source] = ['uid' => $uid, 'path' => $path]; + } + + /** + * Gets the owner and the owner path from the source path + * + * @param string $source source path + * @return array with user id and path + */ + public static function getSourcePathAndUser($source) { + if (isset(self::$sourcePathAndUser[$source])) { + $uid = self::$sourcePathAndUser[$source]['uid']; + $path = self::$sourcePathAndUser[$source]['path']; + unset(self::$sourcePathAndUser[$source]); + } else { + $uid = $path = false; + } + return [$uid, $path]; + } + + /** + * get current size of all versions from a given user + * + * @param string $user user who owns the versions + * @return int versions size + */ + private static function getVersionsSize($user) { + $view = new View('/' . $user); + $fileInfo = $view->getFileInfo('/files_versions'); + return isset($fileInfo['size']) ? $fileInfo['size'] : 0; + } + + /** + * store a new version of a file. + */ + public static function store($filename) { + // if the file gets streamed we need to remove the .part extension + // to get the right target + $ext = pathinfo($filename, PATHINFO_EXTENSION); + if ($ext === 'part') { + $filename = substr($filename, 0, -5); + } + + // we only handle existing files + if (! Filesystem::file_exists($filename) || Filesystem::is_dir($filename)) { + return false; + } + + // since hook paths are always relative to the "default filesystem view" + // we always use the owner from there to get the full node + $uid = Filesystem::getView()->getOwner(''); + + /** @var IRootFolder $rootFolder */ + $rootFolder = Server::get(IRootFolder::class); + $userFolder = $rootFolder->getUserFolder($uid); + + $eventDispatcher = Server::get(IEventDispatcher::class); + try { + $file = $userFolder->get($filename); + } catch (NotFoundException $e) { + return false; + } + + $mount = $file->getMountPoint(); + if ($mount instanceof SharedMount) { + $ownerFolder = $rootFolder->getUserFolder($mount->getShare()->getShareOwner()); + $ownerNode = $ownerFolder->getFirstNodeById($file->getId()); + if ($ownerNode) { + $file = $ownerNode; + $uid = $mount->getShare()->getShareOwner(); + } + } + + /** @var IUserManager $userManager */ + $userManager = Server::get(IUserManager::class); + $user = $userManager->get($uid); + + if (!$user) { + return false; + } + + // no use making versions for empty files + if ($file->getSize() === 0) { + return false; + } + + $event = new CreateVersionEvent($file); + $eventDispatcher->dispatch('OCA\Files_Versions::createVersion', $event); + if ($event->shouldCreateVersion() === false) { + return false; + } + + /** @var IVersionManager $versionManager */ + $versionManager = Server::get(IVersionManager::class); + + $versionManager->createVersion($user, $file); + } + + + /** + * mark file as deleted so that we can remove the versions if the file is gone + * @param string $path + */ + public static function markDeletedFile($path) { + [$uid, $filename] = self::getUidAndFilename($path); + self::$deletedFiles[$path] = [ + 'uid' => $uid, + 'filename' => $filename]; + } + + /** + * delete the version from the storage and cache + * + * @param View $view + * @param string $path + */ + protected static function deleteVersion($view, $path) { + $view->unlink($path); + /** + * @var \OC\Files\Storage\Storage $storage + * @var string $internalPath + */ + [$storage, $internalPath] = $view->resolvePath($path); + $cache = $storage->getCache($internalPath); + $cache->remove($internalPath); + } + + /** + * Delete versions of a file + */ + public static function delete($path) { + $deletedFile = self::$deletedFiles[$path]; + $uid = $deletedFile['uid']; + $filename = $deletedFile['filename']; + + if (!Filesystem::file_exists($path)) { + $view = new View('/' . $uid . '/files_versions'); + + $versions = self::getVersions($uid, $filename); + if (!empty($versions)) { + foreach ($versions as $v) { + \OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]); + self::deleteVersion($view, $filename . '.v' . $v['version']); + \OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]); + } + } + } + unset(self::$deletedFiles[$path]); + } + + /** + * Delete a version of a file + */ + public static function deleteRevision(string $path, int $revision): void { + [$uid, $filename] = self::getUidAndFilename($path); + $view = new View('/' . $uid . '/files_versions'); + \OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $revision, 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]); + self::deleteVersion($view, $filename . '.v' . $revision); + \OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $revision, 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]); + } + + /** + * Rename or copy versions of a file of the given paths + * + * @param string $sourcePath source path of the file to move, relative to + * the currently logged in user's "files" folder + * @param string $targetPath target path of the file to move, relative to + * the currently logged in user's "files" folder + * @param string $operation can be 'copy' or 'rename' + */ + public static function renameOrCopy($sourcePath, $targetPath, $operation) { + [$sourceOwner, $sourcePath] = self::getSourcePathAndUser($sourcePath); + + // it was a upload of a existing file if no old path exists + // in this case the pre-hook already called the store method and we can + // stop here + if ($sourcePath === false) { + return true; + } + + [$targetOwner, $targetPath] = self::getUidAndFilename($targetPath); + + $sourcePath = ltrim($sourcePath, '/'); + $targetPath = ltrim($targetPath, '/'); + + $rootView = new View(''); + + // did we move a directory ? + if ($rootView->is_dir('/' . $targetOwner . '/files/' . $targetPath)) { + // does the directory exists for versions too ? + if ($rootView->is_dir('/' . $sourceOwner . '/files_versions/' . $sourcePath)) { + // create missing dirs if necessary + self::createMissingDirectories($targetPath, new View('/' . $targetOwner)); + + // move the directory containing the versions + $rootView->$operation( + '/' . $sourceOwner . '/files_versions/' . $sourcePath, + '/' . $targetOwner . '/files_versions/' . $targetPath + ); + } + } elseif ($versions = Storage::getVersions($sourceOwner, '/' . $sourcePath)) { + // create missing dirs if necessary + self::createMissingDirectories($targetPath, new View('/' . $targetOwner)); + + foreach ($versions as $v) { + // move each version one by one to the target directory + $rootView->$operation( + '/' . $sourceOwner . '/files_versions/' . $sourcePath . '.v' . $v['version'], + '/' . $targetOwner . '/files_versions/' . $targetPath . '.v' . $v['version'] + ); + } + } + + // if we moved versions directly for a file, schedule expiration check for that file + if (!$rootView->is_dir('/' . $targetOwner . '/files/' . $targetPath)) { + self::scheduleExpire($targetOwner, $targetPath); + } + } + + /** + * Rollback to an old version of a file. + * + * @param string $file file name + * @param int $revision revision timestamp + * @return bool + */ + public static function rollback(string $file, int $revision, IUser $user) { + // add expected leading slash + $filename = '/' . ltrim($file, '/'); + + // Fetch the userfolder to trigger view hooks + $root = Server::get(IRootFolder::class); + $userFolder = $root->getUserFolder($user->getUID()); + + $users_view = new View('/' . $user->getUID()); + $files_view = new View('/' . $user->getUID() . '/files'); + + $versionCreated = false; + + $fileInfo = $files_view->getFileInfo($file); + + // check if user has the permissions to revert a version + if (!$fileInfo->isUpdateable()) { + return false; + } + + //first create a new version + $version = 'files_versions' . $filename . '.v' . $users_view->filemtime('files' . $filename); + if (!$users_view->file_exists($version)) { + $users_view->copy('files' . $filename, 'files_versions' . $filename . '.v' . $users_view->filemtime('files' . $filename)); + $versionCreated = true; + } + + $fileToRestore = 'files_versions' . $filename . '.v' . $revision; + + // Restore encrypted version of the old file for the newly restored file + // This has to happen manually here since the file is manually copied below + $oldVersion = $users_view->getFileInfo($fileToRestore)->getEncryptedVersion(); + $oldFileInfo = $users_view->getFileInfo($fileToRestore); + $cache = $fileInfo->getStorage()->getCache(); + $cache->update( + $fileInfo->getId(), [ + 'encrypted' => $oldVersion, + 'encryptedVersion' => $oldVersion, + 'size' => $oldFileInfo->getData()['size'], + 'unencrypted_size' => $oldFileInfo->getData()['unencrypted_size'], + ] + ); + + // rollback + if (self::copyFileContents($users_view, $fileToRestore, 'files' . $filename)) { + $files_view->touch($file, $revision); + Storage::scheduleExpire($user->getUID(), $file); + + return true; + } elseif ($versionCreated) { + self::deleteVersion($users_view, $version); + } + + return false; + } + + /** + * Stream copy file contents from $path1 to $path2 + * + * @param View $view view to use for copying + * @param string $path1 source file to copy + * @param string $path2 target file + * + * @return bool true for success, false otherwise + */ + private static function copyFileContents($view, $path1, $path2) { + /** @var \OC\Files\Storage\Storage $storage1 */ + [$storage1, $internalPath1] = $view->resolvePath($path1); + /** @var \OC\Files\Storage\Storage $storage2 */ + [$storage2, $internalPath2] = $view->resolvePath($path2); + + $view->lockFile($path1, ILockingProvider::LOCK_EXCLUSIVE); + $view->lockFile($path2, ILockingProvider::LOCK_EXCLUSIVE); + + try { + // TODO add a proper way of overwriting a file while maintaining file ids + if ($storage1->instanceOfStorage(ObjectStoreStorage::class) + || $storage2->instanceOfStorage(ObjectStoreStorage::class) + ) { + $source = $storage1->fopen($internalPath1, 'r'); + $result = $source !== false; + if ($result) { + if ($storage2->instanceOfStorage(IWriteStreamStorage::class)) { + /** @var IWriteStreamStorage $storage2 */ + $storage2->writeStream($internalPath2, $source); + } else { + $target = $storage2->fopen($internalPath2, 'w'); + $result = $target !== false; + if ($result) { + [, $result] = Files::streamCopy($source, $target, true); + } + // explicit check as S3 library closes streams already + if (is_resource($target)) { + fclose($target); + } + } + } + // explicit check as S3 library closes streams already + if (is_resource($source)) { + fclose($source); + } + + if ($result !== false) { + $storage1->unlink($internalPath1); + } + } else { + $result = $storage2->moveFromStorage($storage1, $internalPath1, $internalPath2); + } + } finally { + $view->unlockFile($path1, ILockingProvider::LOCK_EXCLUSIVE); + $view->unlockFile($path2, ILockingProvider::LOCK_EXCLUSIVE); + } + + return ($result !== false); + } + + /** + * get a list of all available versions of a file in descending chronological order + * @param string $uid user id from the owner of the file + * @param string $filename file to find versions of, relative to the user files dir + * @param string $userFullPath + * @return array versions newest version first + */ + public static function getVersions($uid, $filename, $userFullPath = '') { + $versions = []; + if (empty($filename)) { + return $versions; + } + // fetch for old versions + $view = new View('/' . $uid . '/'); + + $pathinfo = pathinfo($filename); + $versionedFile = $pathinfo['basename']; + + $dir = Filesystem::normalizePath(self::VERSIONS_ROOT . '/' . $pathinfo['dirname']); + + $dirContent = false; + if ($view->is_dir($dir)) { + $dirContent = $view->opendir($dir); + } + + if ($dirContent === false) { + return $versions; + } + + if (is_resource($dirContent)) { + while (($entryName = readdir($dirContent)) !== false) { + if (!Filesystem::isIgnoredDir($entryName)) { + $pathparts = pathinfo($entryName); + $filename = $pathparts['filename']; + if ($filename === $versionedFile) { + $pathparts = pathinfo($entryName); + $timestamp = substr($pathparts['extension'] ?? '', 1); + if (!is_numeric($timestamp)) { + Server::get(LoggerInterface::class)->error( + 'Version file {path} has incorrect name format', + [ + 'path' => $entryName, + 'app' => 'files_versions', + ] + ); + continue; + } + $filename = $pathparts['filename']; + $key = $timestamp . '#' . $filename; + $versions[$key]['version'] = $timestamp; + $versions[$key]['humanReadableTimestamp'] = self::getHumanReadableTimestamp((int)$timestamp); + if (empty($userFullPath)) { + $versions[$key]['preview'] = ''; + } else { + /** @var IURLGenerator $urlGenerator */ + $urlGenerator = Server::get(IURLGenerator::class); + $versions[$key]['preview'] = $urlGenerator->linkToRoute('files_version.Preview.getPreview', + ['file' => $userFullPath, 'version' => $timestamp]); + } + $versions[$key]['path'] = Filesystem::normalizePath($pathinfo['dirname'] . '/' . $filename); + $versions[$key]['name'] = $versionedFile; + $versions[$key]['size'] = $view->filesize($dir . '/' . $entryName); + $versions[$key]['mimetype'] = Server::get(IMimeTypeDetector::class)->detectPath($versionedFile); + } + } + } + closedir($dirContent); + } + + // sort with newest version first + krsort($versions); + + return $versions; + } + + /** + * Expire versions that older than max version retention time + * + * @param string $uid + */ + public static function expireOlderThanMaxForUser($uid) { + /** @var IRootFolder $root */ + $root = Server::get(IRootFolder::class); + try { + /** @var Folder $versionsRoot */ + $versionsRoot = $root->get('/' . $uid . '/files_versions'); + } catch (NotFoundException $e) { + return; + } + + $expiration = self::getExpiration(); + $threshold = $expiration->getMaxAgeAsTimestamp(); + if (!$threshold) { + return; + } + + $allVersions = $versionsRoot->search(new SearchQuery( + new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_NOT, [ + new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', FileInfo::MIMETYPE_FOLDER), + ]), + 0, + 0, + [] + )); + + /** @var VersionsMapper $versionsMapper */ + $versionsMapper = Server::get(VersionsMapper::class); + $userFolder = $root->getUserFolder($uid); + $versionEntities = []; + + /** @var Node[] $versions */ + $versions = array_filter($allVersions, function (Node $info) use ($threshold, $userFolder, $versionsMapper, $versionsRoot, &$versionEntities) { + // Check that the file match '*.v*' + $versionsBegin = strrpos($info->getName(), '.v'); + if ($versionsBegin === false) { + return false; + } + + $version = (int)substr($info->getName(), $versionsBegin + 2); + + // Check that the version does not have a label. + $path = $versionsRoot->getRelativePath($info->getPath()); + if ($path === null) { + throw new DoesNotExistException('Could not find relative path of (' . $info->getPath() . ')'); + } + + try { + $node = $userFolder->get(substr($path, 0, -strlen('.v' . $version))); + $versionEntity = $versionsMapper->findVersionForFileId($node->getId(), $version); + $versionEntities[$info->getId()] = $versionEntity; + + if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') { + return false; + } + } catch (NotFoundException $e) { + // Original node not found, delete the version + return true; + } catch (StorageNotAvailableException|StorageInvalidException $e) { + // Storage can't be used, but it might only be temporary so we can't always delete the version + // since we can't determine if the version is named we take the safe route and don't expire + return false; + } catch (DoesNotExistException $ex) { + // Version on FS can have no equivalent in the DB if they were created before the version naming feature. + // So we ignore DoesNotExistException. + } + + // Check that the version's timestamp is lower than $threshold + return $version < $threshold; + }); + + foreach ($versions as $version) { + $internalPath = $version->getInternalPath(); + \OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]); + + $versionEntity = isset($versionEntities[$version->getId()]) ? $versionEntities[$version->getId()] : null; + if (!is_null($versionEntity)) { + $versionsMapper->delete($versionEntity); + } + + try { + $version->delete(); + \OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]); + } catch (NotPermittedException $e) { + Server::get(LoggerInterface::class)->error("Missing permissions to delete version: {$internalPath}", ['app' => 'files_versions', 'exception' => $e]); + } + } + } + + /** + * translate a timestamp into a string like "5 days ago" + * + * @param int $timestamp + * @return string for example "5 days ago" + */ + private static function getHumanReadableTimestamp(int $timestamp): string { + $diff = time() - $timestamp; + + if ($diff < 60) { // first minute + return $diff . ' seconds ago'; + } elseif ($diff < 3600) { //first hour + return round($diff / 60) . ' minutes ago'; + } elseif ($diff < 86400) { // first day + return round($diff / 3600) . ' hours ago'; + } elseif ($diff < 604800) { //first week + return round($diff / 86400) . ' days ago'; + } elseif ($diff < 2419200) { //first month + return round($diff / 604800) . ' weeks ago'; + } elseif ($diff < 29030400) { // first year + return round($diff / 2419200) . ' months ago'; + } else { + return round($diff / 29030400) . ' years ago'; + } + } + + /** + * returns all stored file versions from a given user + * @param string $uid id of the user + * @return array with contains two arrays 'all' which contains all versions sorted by age and 'by_file' which contains all versions sorted by filename + */ + private static function getAllVersions($uid) { + $view = new View('/' . $uid . '/'); + $dirs = [self::VERSIONS_ROOT]; + $versions = []; + + while (!empty($dirs)) { + $dir = array_pop($dirs); + $files = $view->getDirectoryContent($dir); + + foreach ($files as $file) { + $fileData = $file->getData(); + $filePath = $dir . '/' . $fileData['name']; + if ($file['type'] === 'dir') { + $dirs[] = $filePath; + } else { + $versionsBegin = strrpos($filePath, '.v'); + $relPathStart = strlen(self::VERSIONS_ROOT); + $version = substr($filePath, $versionsBegin + 2); + $relpath = substr($filePath, $relPathStart, $versionsBegin - $relPathStart); + $key = $version . '#' . $relpath; + $versions[$key] = ['path' => $relpath, 'timestamp' => $version]; + } + } + } + + // newest version first + krsort($versions); + + $result = [ + 'all' => [], + 'by_file' => [], + ]; + + foreach ($versions as $key => $value) { + $size = $view->filesize(self::VERSIONS_ROOT . '/' . $value['path'] . '.v' . $value['timestamp']); + $filename = $value['path']; + + $result['all'][$key]['version'] = $value['timestamp']; + $result['all'][$key]['path'] = $filename; + $result['all'][$key]['size'] = $size; + + $result['by_file'][$filename][$key]['version'] = $value['timestamp']; + $result['by_file'][$filename][$key]['path'] = $filename; + $result['by_file'][$filename][$key]['size'] = $size; + } + + return $result; + } + + /** + * get list of files we want to expire + * @param array $versions list of versions + * @param integer $time + * @param bool $quotaExceeded is versions storage limit reached + * @return array containing the list of to deleted versions and the size of them + */ + protected static function getExpireList($time, $versions, $quotaExceeded = false) { + $expiration = self::getExpiration(); + + if ($expiration->shouldAutoExpire()) { + // Exclude versions that are newer than the minimum age from the auto expiration logic. + $minAge = $expiration->getMinAgeAsTimestamp(); + if ($minAge !== false) { + $versionsToAutoExpire = array_filter($versions, fn ($version) => $version['version'] < $minAge); + } else { + $versionsToAutoExpire = $versions; + } + + [$toDelete, $size] = self::getAutoExpireList($time, $versionsToAutoExpire); + } else { + $size = 0; + $toDelete = []; // versions we want to delete + } + + foreach ($versions as $key => $version) { + if (!is_numeric($version['version'])) { + Server::get(LoggerInterface::class)->error( + 'Found a non-numeric timestamp version: ' . json_encode($version), + ['app' => 'files_versions']); + continue; + } + if ($expiration->isExpired((int)($version['version']), $quotaExceeded) && !isset($toDelete[$key])) { + $size += $version['size']; + $toDelete[$key] = $version['path'] . '.v' . $version['version']; + } + } + + return [$toDelete, $size]; + } + + /** + * get list of files we want to expire + * @param array $versions list of versions + * @param integer $time + * @return array containing the list of to deleted versions and the size of them + */ + protected static function getAutoExpireList($time, $versions) { + $size = 0; + $toDelete = []; // versions we want to delete + + $interval = 1; + $step = Storage::$max_versions_per_interval[$interval]['step']; + if (Storage::$max_versions_per_interval[$interval]['intervalEndsAfter'] === -1) { + $nextInterval = -1; + } else { + $nextInterval = $time - Storage::$max_versions_per_interval[$interval]['intervalEndsAfter']; + } + + $firstVersion = reset($versions); + + if ($firstVersion === false) { + return [$toDelete, $size]; + } + + $firstKey = key($versions); + $prevTimestamp = $firstVersion['version']; + $nextVersion = $firstVersion['version'] - $step; + unset($versions[$firstKey]); + + foreach ($versions as $key => $version) { + $newInterval = true; + while ($newInterval) { + if ($nextInterval === -1 || $prevTimestamp > $nextInterval) { + if ($version['version'] > $nextVersion) { + //distance between two version too small, mark to delete + $toDelete[$key] = $version['path'] . '.v' . $version['version']; + $size += $version['size']; + Server::get(LoggerInterface::class)->info('Mark to expire ' . $version['path'] . ' next version should be ' . $nextVersion . ' or smaller. (prevTimestamp: ' . $prevTimestamp . '; step: ' . $step, ['app' => 'files_versions']); + } else { + $nextVersion = $version['version'] - $step; + $prevTimestamp = $version['version']; + } + $newInterval = false; // version checked so we can move to the next one + } else { // time to move on to the next interval + $interval++; + $step = Storage::$max_versions_per_interval[$interval]['step']; + $nextVersion = $prevTimestamp - $step; + if (Storage::$max_versions_per_interval[$interval]['intervalEndsAfter'] === -1) { + $nextInterval = -1; + } else { + $nextInterval = $time - Storage::$max_versions_per_interval[$interval]['intervalEndsAfter']; + } + $newInterval = true; // we changed the interval -> check same version with new interval + } + } + } + + return [$toDelete, $size]; + } + + /** + * Schedule versions expiration for the given file + * + * @param string $uid owner of the file + * @param string $fileName file/folder for which to schedule expiration + */ + public static function scheduleExpire($uid, $fileName) { + // let the admin disable auto expire + $expiration = self::getExpiration(); + if ($expiration->isEnabled()) { + $command = new Expire($uid, $fileName); + /** @var IBus $bus */ + $bus = Server::get(IBus::class); + $bus->push($command); + } + } + + /** + * Expire versions which exceed the quota. + * + * This will setup the filesystem for the given user but will not + * tear it down afterwards. + * + * @param string $filename path to file to expire + * @param string $uid user for which to expire the version + * @return bool|int|null + */ + public static function expire($filename, $uid) { + $expiration = self::getExpiration(); + + /** @var LoggerInterface $logger */ + $logger = Server::get(LoggerInterface::class); + + if ($expiration->isEnabled()) { + // get available disk space for user + $user = Server::get(IUserManager::class)->get($uid); + if (is_null($user)) { + $logger->error('Backends provided no user object for ' . $uid, ['app' => 'files_versions']); + throw new NoUserException('Backends provided no user object for ' . $uid); + } + + \OC_Util::setupFS($uid); + + try { + if (!Filesystem::file_exists($filename)) { + return false; + } + } catch (StorageNotAvailableException $e) { + // if we can't check that the file hasn't been deleted we can only assume that it hasn't + // note that this `StorageNotAvailableException` is about the file the versions originate from, + // not the storage that the versions are stored on + } + + if (empty($filename)) { + // file maybe renamed or deleted + return false; + } + $versionsFileview = new View('/' . $uid . '/files_versions'); + + $softQuota = true; + $quota = $user->getQuota(); + if ($quota === null || $quota === 'none') { + $quota = Filesystem::free_space('/'); + $softQuota = false; + } else { + $quota = Util::computerFileSize($quota); + } + + // make sure that we have the current size of the version history + $versionsSize = self::getVersionsSize($uid); + + // calculate available space for version history + // subtract size of files and current versions size from quota + if ($quota >= 0) { + if ($softQuota) { + $root = Server::get(IRootFolder::class); + $userFolder = $root->getUserFolder($uid); + if (is_null($userFolder)) { + $availableSpace = 0; + } else { + $free = $quota - $userFolder->getSize(false); // remaining free space for user + if ($free > 0) { + $availableSpace = ($free * self::DEFAULTMAXSIZE / 100) - $versionsSize; // how much space can be used for versions + } else { + $availableSpace = $free - $versionsSize; + } + } + } else { + $availableSpace = $quota; + } + } else { + $availableSpace = PHP_INT_MAX; + } + + $allVersions = Storage::getVersions($uid, $filename); + + $time = time(); + [$toDelete, $sizeOfDeletedVersions] = self::getExpireList($time, $allVersions, $availableSpace <= 0); + + $availableSpace = $availableSpace + $sizeOfDeletedVersions; + $versionsSize = $versionsSize - $sizeOfDeletedVersions; + + // if still not enough free space we rearrange the versions from all files + if ($availableSpace <= 0) { + $result = self::getAllVersions($uid); + $allVersions = $result['all']; + + foreach ($result['by_file'] as $versions) { + [$toDeleteNew, $size] = self::getExpireList($time, $versions, $availableSpace <= 0); + $toDelete = array_merge($toDelete, $toDeleteNew); + $sizeOfDeletedVersions += $size; + } + $availableSpace = $availableSpace + $sizeOfDeletedVersions; + $versionsSize = $versionsSize - $sizeOfDeletedVersions; + } + + foreach ($toDelete as $key => $path) { + // Make sure to cleanup version table relations as expire does not pass deleteVersion + try { + /** @var VersionsMapper $versionsMapper */ + $versionsMapper = Server::get(VersionsMapper::class); + $file = Server::get(IRootFolder::class)->getUserFolder($uid)->get($filename); + $pathparts = pathinfo($path); + $timestamp = (int)substr($pathparts['extension'] ?? '', 1); + $versionEntity = $versionsMapper->findVersionForFileId($file->getId(), $timestamp); + if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') { + continue; + } + $versionsMapper->delete($versionEntity); + } catch (DoesNotExistException $e) { + } + + \OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]); + self::deleteVersion($versionsFileview, $path); + \OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]); + unset($allVersions[$key]); // update array with the versions we keep + $logger->info('Expire: ' . $path, ['app' => 'files_versions']); + } + + // Check if enough space is available after versions are rearranged. + // If not we delete the oldest versions until we meet the size limit for versions, + // but always keep the two latest versions + $numOfVersions = count($allVersions) - 2 ; + $i = 0; + // sort oldest first and make sure that we start at the first element + ksort($allVersions); + reset($allVersions); + while ($availableSpace < 0 && $i < $numOfVersions) { + $version = current($allVersions); + \OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]); + self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']); + \OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]); + $logger->info('running out of space! Delete oldest version: ' . $version['path'] . '.v' . $version['version'], ['app' => 'files_versions']); + $versionsSize -= $version['size']; + $availableSpace += $version['size']; + next($allVersions); + $i++; + } + + return $versionsSize; // finally return the new size of the version history + } + + return false; + } + + /** + * Create recursively missing directories inside of files_versions + * that match the given path to a file. + * + * @param string $filename $path to a file, relative to the user's + * "files" folder + * @param View $view view on data/user/ + */ + public static function createMissingDirectories($filename, $view) { + $dirname = Filesystem::normalizePath(dirname($filename)); + $dirParts = explode('/', $dirname); + $dir = '/files_versions'; + foreach ($dirParts as $part) { + $dir = $dir . '/' . $part; + if (!$view->file_exists($dir)) { + $view->mkdir($dir); + } + } + } + + /** + * Static workaround + * @return Expiration + */ + protected static function getExpiration() { + if (self::$application === null) { + self::$application = Server::get(Application::class); + } + return self::$application->getContainer()->get(Expiration::class); + } +} |