* * @author Andreas Fischer * @author Lukas Reschke * @author Michael Roitzsch * @author Morris Jobke * @author Thomas Müller * * @license AGPL-3.0 * * This code is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, * as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License, version 3, * along with this program. If not, see * */ namespace OC; /** * Helper class for large files on 32-bit platforms. */ class LargeFileHelper { /** * pow(2, 53) as a base-10 string. * @var string */ const POW_2_53 = '9007199254740992'; /** * pow(2, 53) - 1 as a base-10 string. * @var string */ const POW_2_53_MINUS_1 = '9007199254740991'; /** * @brief Checks whether our assumptions hold on the PHP platform we are on. * * @throws \RunTimeException if our assumptions do not hold on the current * PHP platform. */ public function __construct() { $pow_2_53 = (float)self::POW_2_53_MINUS_1 + 1.0; if ($this->formatUnsignedInteger($pow_2_53) !== self::POW_2_53) { throw new \RuntimeException( 'This class assumes floats to be double precision or "better".' ); } } /** * @brief Formats a signed integer or float as an unsigned integer base-10 * string. Passed strings will be checked for being base-10. * * @param int|float|string $number Number containing unsigned integer data * * @throws \UnexpectedValueException if $number is not a float, not an int * and not a base-10 string. * * @return string Unsigned integer base-10 string */ public function formatUnsignedInteger($number) { if (is_float($number)) { // Undo the effect of the php.ini setting 'precision'. return number_format($number, 0, '', ''); } else if (is_string($number) && ctype_digit($number)) { return $number; } else if (is_int($number)) { // Interpret signed integer as unsigned integer. return sprintf('%u', $number); } else { throw new \UnexpectedValueException( 'Expected int, float or base-10 string' ); } } /** * @brief Tries to get the size of a file via various workarounds that * even work for large files on 32-bit platforms. * * @param string $filename Path to the file. * * @return null|int|float Number of bytes as number (float or int) or * null on failure. */ public function getFileSize($filename) { $fileSize = $this->getFileSizeViaCurl($filename); if (!is_null($fileSize)) { return $fileSize; } $fileSize = $this->getFileSizeViaExec($filename); if (!is_null($fileSize)) { return $fileSize; } return $this->getFileSizeNative($filename); } /** * @brief Tries to get the size of a file via a CURL HEAD request. * * @param string $fileName Path to the file. * * @return null|int|float Number of bytes as number (float or int) or * null on failure. */ public function getFileSizeViaCurl($fileName) { if (\OC::$server->getIniWrapper()->getString('open_basedir') === '') { $encodedFileName = rawurlencode($fileName); $ch = curl_init("file://$encodedFileName"); curl_setopt($ch, CURLOPT_NOBODY, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_HEADER, true); $data = curl_exec($ch); curl_close($ch); if ($data !== false) { $matches = array(); preg_match('/Content-Length: (\d+)/', $data, $matches); if (isset($matches[1])) { return 0 + $matches[1]; } } } return null; } /** * @brief Tries to get the size of a file via an exec() call. * * @param string $filename Path to the file. * * @return null|int|float Number of bytes as number (float or int) or * null on failure. */ public function getFileSizeViaExec($filename) { if (\OC_Helper::is_function_enabled('exec')) { $os = strtolower(php_uname('s')); $arg = escapeshellarg($filename); $result = null; if (strpos($os, 'linux') !== false) { $result = $this->exec("stat -c %s $arg"); } else if (strpos($os, 'bsd') !== false || strpos($os, 'darwin') !== false) { $result = $this->exec("stat -f %z $arg"); } return $result; } return null; } /** * @brief Gets the size of a file via a filesize() call and converts * negative signed int to positive float. As the result of filesize() * will wrap around after a file size of 2^32 bytes = 4 GiB, this * should only be used as a last resort. * * @param string $filename Path to the file. * * @return int|float Number of bytes as number (float or int). */ public function getFileSizeNative($filename) { $result = filesize($filename); if ($result < 0) { // For file sizes between 2 GiB and 4 GiB, filesize() will return a // negative int, as the PHP data type int is signed. Interpret the // returned int as an unsigned integer and put it into a float. return (float) sprintf('%u', $result); } return $result; } /** * Returns the current mtime for $fullPath * * @param string $fullPath * @return int */ public function getFileMtime($fullPath) { if (\OC_Helper::is_function_enabled('exec')) { $os = strtolower(php_uname('s')); if (strpos($os, 'linux') !== false) { return $this->exec('stat -c %Y ' . escapeshellarg($fullPath)); } } return filemtime($fullPath); } protected function exec($cmd) { $result = trim(exec($cmd)); return ctype_digit($result) ? 0 + $result : null; } } parent; padding-left: 5px; padding-right: 5px; } td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } .highlight .hll { background-color: #ffffcc } .highlight .c { color: #888888 } /* Comment */ .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ .highlight .k { color: #008800; font-weight: bold } /* Keyword */ .highlight .ch { color: #888888 } /* Comment.Hashbang */ .highlight .cm { color: #888888 } /* Comment.Multiline */ .highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */ .highlight .cpf { color: #888888 } /* Comment.PreprocFile */ .highlight .c1 { color: #888888 } /* Comment.Single */ .highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */ .highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gr { color: #aa0000 } /* Generic.Error */ .highlight .gh { color: #333333 } /* Generic.Heading */ .highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #555555 } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #666666 } /* Generic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package bleve

import (
	"github.com/blevesearch/bleve/v2"
)

// FlushingBatch is a batch of operations that automatically flushes to the
// underlying index once it reaches a certain size.
type FlushingBatch struct {
	maxBatchSize int
	batch        *bleve.Batch
	index        bleve.Index
}

// NewFlushingBatch creates a new flushing batch for the specified index. Once
// the number of operations in the batch reaches the specified limit, the batch
// automatically flushes its operations to the index.
func NewFlushingBatch(index bleve.Index, maxBatchSize int) *FlushingBatch {
	return &FlushingBatch{
		maxBatchSize: maxBatchSize,
		batch:        index.NewBatch(),
		index:        index,
	}
}

// Index add a new index to batch
func (b *FlushingBatch) Index(id string, data interface{}) error {
	if err := b.batch.Index(id, data); err != nil {
		return err
	}
	return b.flushIfFull()
}

// Delete add a delete index to batch
func (b *FlushingBatch) Delete(id string) error {
	b.batch.Delete(id)
	return b.flushIfFull()
}

func (b *FlushingBatch) flushIfFull() error {
	if b.batch.Size() < b.maxBatchSize {
		return nil
	}
	return b.Flush()
}

// Flush submit the batch and create a new one
func (b *FlushingBatch) Flush() error {
	err := b.index.Batch(b.batch)
	if err != nil {
		return err
	}
	b.batch = b.index.NewBatch()
	return nil
}