summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/workflows/sftp.yml75
-rw-r--r--apps/files_external/lib/Lib/Storage/SFTP.php125
-rw-r--r--apps/files_external/lib/Lib/Storage/SFTPReadStream.php35
3 files changed, 226 insertions, 9 deletions
diff --git a/.github/workflows/sftp.yml b/.github/workflows/sftp.yml
new file mode 100644
index 00000000000..939372a53d2
--- /dev/null
+++ b/.github/workflows/sftp.yml
@@ -0,0 +1,75 @@
+name: SFTP unit tests
+on:
+ push:
+ branches:
+ - master
+ - stable*
+ paths:
+ - 'apps/files_external/**'
+ pull_request:
+ paths:
+ - 'apps/files_external/**'
+
+env:
+ APP_NAME: files_external
+
+jobs:
+ sftp-tests:
+ runs-on: ubuntu-latest
+
+ if: ${{ github.repository_owner != 'nextcloud-gmbh' }}
+
+ strategy:
+ # do not stop on another job's failure
+ fail-fast: false
+ matrix:
+ php-versions: ['8.0']
+ sftpd: ['openssh']
+
+ name: php${{ matrix.php-versions }}-${{ matrix.sftpd }}
+
+ steps:
+ - name: Checkout server
+ uses: actions/checkout@v3
+ with:
+ submodules: true
+
+ - name: Set up sftpd
+ run: |
+ sudo mkdir /tmp/sftp
+ sudo chown -R 0777 /tmp/sftp
+ if [[ "${{ matrix.sftpd }}" == 'openssh' ]]; then docker run -p 2222:22 --name sftp -d -v /tmp/sftp:/home/test atmoz/sftp "test:test:::data"; fi
+ - name: Set up php ${{ matrix.php-versions }}
+ uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2
+ with:
+ php-version: ${{ matrix.php-versions }}
+ tools: phpunit:9
+ extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set up Nextcloud
+ run: |
+ mkdir data
+ ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
+ ./occ app:enable --force ${{ env.APP_NAME }}
+ php -S localhost:8080 &
+ - name: PHPUnit
+ run: |
+ echo "<?php return ['run' => true, 'host' => 'localhost:2222','user' => 'test','password' => 'test', 'root' => 'data'];" > apps/${{ env.APP_NAME }}/tests/config.sftp.php
+ phpunit --configuration tests/phpunit-autotest-external.xml apps/files_external/tests/Storage/SftpTest.php
+ - name: sftpd logs
+ if: always()
+ run: |
+ ls -l /tmp/sftp
+ docker logs sftp
+
+ sftp-summary:
+ runs-on: ubuntu-latest
+ needs: sftp-tests
+
+ if: always()
+
+ steps:
+ - name: Summary status
+ run: if ${{ needs.sftp-tests.result != 'success' }}; then exit 1; fi
diff --git a/apps/files_external/lib/Lib/Storage/SFTP.php b/apps/files_external/lib/Lib/Storage/SFTP.php
index 532c50808db..5c4dcca65ce 100644
--- a/apps/files_external/lib/Lib/Storage/SFTP.php
+++ b/apps/files_external/lib/Lib/Storage/SFTP.php
@@ -36,15 +36,21 @@
*/
namespace OCA\Files_External\Lib\Storage;
+use Icewind\Streams\CountWrapper;
use Icewind\Streams\IteratorDirectory;
use Icewind\Streams\RetryWrapper;
+use OC\Files\Filesystem;
+use OC\Files\Storage\Common;
+use OCP\Constants;
+use OCP\Files\FileInfo;
+use OCP\Files\IMimeTypeDetector;
use phpseclib\Net\SFTP\Stream;
/**
* Uses phpseclib's Net\SFTP class and the Net\SFTP\Stream stream wrapper to
* provide access to SFTP servers.
*/
-class SFTP extends \OC\Files\Storage\Common {
+class SFTP extends Common {
private $host;
private $user;
private $root;
@@ -56,6 +62,9 @@ class SFTP extends \OC\Files\Storage\Common {
* @var \phpseclib\Net\SFTP
*/
protected $client;
+ private IMimeTypeDetector $mimeTypeDetector;
+
+ const COPY_CHUNK_SIZE = 8 * 1024 * 1024;
/**
* @param string $host protocol://server:port
@@ -111,6 +120,7 @@ class SFTP extends \OC\Files\Storage\Common {
$this->root = '/' . ltrim($this->root, '/');
$this->root = rtrim($this->root, '/') . '/';
+ $this->mimeTypeDetector = \OC::$server->get(IMimeTypeDetector::class);
}
/**
@@ -370,20 +380,24 @@ class SFTP extends \OC\Files\Storage\Common {
public function fopen($path, $mode) {
try {
$absPath = $this->absPath($path);
+ $connection = $this->getConnection();
switch ($mode) {
case 'r':
case 'rb':
- if (!$this->file_exists($path)) {
+ $stat = $this->stat($path);
+ if (!$stat) {
return false;
}
SFTPReadStream::register();
- $context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]);
+ $context = stream_context_create(['sftp' => ['session' => $connection, 'size' => $stat['size']]]);
$handle = fopen('sftpread://' . trim($absPath, '/'), 'r', false, $context);
return RetryWrapper::wrap($handle);
case 'w':
case 'wb':
SFTPWriteStream::register();
- $context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]);
+ // the SFTPWriteStream doesn't go through the "normal" methods so it doesn't clear the stat cache.
+ $connection->_remove_from_stat_cache($absPath);
+ $context = stream_context_create(['sftp' => ['session' => $connection]]);
return fopen('sftpwrite://' . trim($absPath, '/'), 'w', false, $context);
case 'a':
case 'ab':
@@ -395,7 +409,7 @@ class SFTP extends \OC\Files\Storage\Common {
case 'x+':
case 'c':
case 'c+':
- $context = stream_context_create(['sftp' => ['session' => $this->getConnection()]]);
+ $context = stream_context_create(['sftp' => ['session' => $connection]]);
$handle = fopen($this->constructUrl($path), $mode, false, $context);
return RetryWrapper::wrap($handle);
}
@@ -450,14 +464,14 @@ class SFTP extends \OC\Files\Storage\Common {
}
/**
- * {@inheritdoc}
+ * @return array{mtime: int, size: int, ctime: int}|false
*/
public function stat($path) {
try {
$stat = $this->getConnection()->stat($this->absPath($path));
- $mtime = $stat ? $stat['mtime'] : -1;
- $size = $stat ? $stat['size'] : 0;
+ $mtime = $stat ? (int)$stat['mtime'] : -1;
+ $size = $stat ? (int)$stat['size'] : 0;
return ['mtime' => $mtime, 'size' => $size, 'ctime' => -1];
} catch (\Exception $e) {
@@ -476,4 +490,99 @@ class SFTP extends \OC\Files\Storage\Common {
$url = 'sftp://' . urlencode($this->user) . '@' . $this->host . ':' . $this->port . $this->root . $path;
return $url;
}
+
+ public function file_put_contents($path, $data) {
+ /** @psalm-suppress InternalMethod */
+ $result = $this->getConnection()->put($this->absPath($path), $data);
+ if ($result) {
+ return strlen($data);
+ } else {
+ return false;
+ }
+ }
+
+ public function writeStream(string $path, $stream, int $size = null): int {
+ if ($size === null) {
+ $stream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size) {
+ $size = $writtenSize;
+ });
+ if (!$stream) {
+ throw new \Exception("Failed to wrap stream");
+ }
+ }
+ /** @psalm-suppress InternalMethod */
+ $result = $this->getConnection()->put($this->absPath($path), $stream);
+ fclose($stream);
+ if ($result) {
+ return $size;
+ } else {
+ throw new \Exception("Failed to write steam to sftp storage");
+ }
+ }
+
+ public function copy($source, $target) {
+ if ($this->is_dir($source) || $this->is_dir($target)) {
+ return parent::copy($source, $target);
+ } else {
+ $absSource = $this->absPath($source);
+ $absTarget = $this->absPath($target);
+
+ $connection = $this->getConnection();
+ $size = $connection->size($absSource);
+ if ($size === false) {
+ return false;
+ }
+ for ($i = 0; $i < $size; $i += self::COPY_CHUNK_SIZE) {
+ /** @psalm-suppress InvalidArgument */
+ $chunk = $connection->get($absSource, false, $i, self::COPY_CHUNK_SIZE);
+ if ($chunk === false) {
+ return false;
+ }
+ /** @psalm-suppress InternalMethod */
+ if (!$connection->put($absTarget, $chunk, \phpseclib\Net\SFTP::SOURCE_STRING, $i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+ public function getPermissions($path) {
+ $stat = $this->getConnection()->stat($this->absPath($path));
+ if (!$stat) {
+ return 0;
+ }
+ if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) {
+ return Constants::PERMISSION_ALL;
+ } else {
+ return Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
+ }
+ }
+
+ public function getMetaData($path) {
+ $stat = $this->getConnection()->stat($this->absPath($path));
+ if (!$stat) {
+ return null;
+ }
+
+ if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) {
+ $stat['permissions'] = Constants::PERMISSION_ALL;
+ } else {
+ $stat['permissions'] = Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
+ }
+
+ if ($stat['type'] === NET_SFTP_TYPE_DIRECTORY) {
+ $stat['size'] = -1;
+ $stat['mimetype'] = FileInfo::MIMETYPE_FOLDER;
+ } else {
+ $stat['mimetype'] = $this->mimeTypeDetector->detectPath($path);
+ }
+
+ $stat['etag'] = $this->getETag($path);
+ $stat['storage_mtime'] = $stat['mtime'];
+ $stat['name'] = basename($path);
+
+ $keys = ['size', 'mtime', 'mimetype', 'etag', 'storage_mtime', 'permissions', 'name'];
+ return array_intersect_key($stat, array_flip($keys));
+ }
}
diff --git a/apps/files_external/lib/Lib/Storage/SFTPReadStream.php b/apps/files_external/lib/Lib/Storage/SFTPReadStream.php
index c4749b15453..7a98c6b2a6d 100644
--- a/apps/files_external/lib/Lib/Storage/SFTPReadStream.php
+++ b/apps/files_external/lib/Lib/Storage/SFTPReadStream.php
@@ -49,6 +49,8 @@ class SFTPReadStream implements File {
private $eof = false;
private $buffer = '';
+ private bool $pendingRead = false;
+ private int $size = 0;
public static function register($protocol = 'sftpread') {
if (in_array($protocol, stream_get_wrappers(), true)) {
@@ -75,6 +77,9 @@ class SFTPReadStream implements File {
} else {
throw new \BadMethodCallException('Invalid context, session not set');
}
+ if (isset($context['size'])) {
+ $this->size = $context['size'];
+ }
return $context;
}
@@ -118,7 +123,25 @@ class SFTPReadStream implements File {
}
public function stream_seek($offset, $whence = SEEK_SET) {
- return false;
+ switch ($whence) {
+ case SEEK_SET:
+ $this->seekTo($offset);
+ break;
+ case SEEK_CUR:
+ $this->seekTo($this->readPosition + $offset);
+ break;
+ case SEEK_END:
+ $this->seekTo($this->size + $offset);
+ break;
+ }
+ return true;
+ }
+
+ private function seekTo(int $offset): void {
+ $this->internalPosition = $offset;
+ $this->readPosition = $offset;
+ $this->buffer = '';
+ $this->request_chunk(256 * 1024);
}
public function stream_tell() {
@@ -142,11 +165,17 @@ class SFTPReadStream implements File {
}
private function request_chunk($size) {
+ if ($this->pendingRead) {
+ $this->sftp->_get_sftp_packet();
+ }
+
$packet = pack('Na*N3', strlen($this->handle), $this->handle, $this->internalPosition / 4294967296, $this->internalPosition, $size);
+ $this->pendingRead = true;
return $this->sftp->_send_sftp_packet(NET_SFTP_READ, $packet);
}
private function read_chunk() {
+ $this->pendingRead = false;
$response = $this->sftp->_get_sftp_packet();
switch ($this->sftp->packet_type) {
@@ -195,6 +224,10 @@ class SFTPReadStream implements File {
}
public function stream_close() {
+ // we still have a read request incoming that needs to be handled before we can close
+ if ($this->pendingRead) {
+ $this->sftp->_get_sftp_packet();
+ }
if (!$this->sftp->_close_handle($this->handle)) {
return false;
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-06 08:19:13 +0000