Simplify ReftableCompactor
The ReftableCompactor supported a byteLimit, but this is currently
unused. The FileReftableStack has a more sophisticated strategy that
amortizes compaction costs.
Rename min/maxUpdateIndex to reflogExpire{Min,Max}UpdateIndex to
reflect their purpose more accurately.
Since reflogs are generally pruned chronologically (oldest entries are
expired first), one can only prune entries on full compaction, so they
should not be set by default.
Rephrase the function Reader#minUpdateIndex and maxUpdateIndex. These
vars are documented to affect log entries, but semantically, they are
about ref entries. Since ref entries have their timestamps
delta-compressed, it is important for the min/maxUpdateIndex values to
be coherent between different tables.
The logical timestamps for log entries do not have to be coherent in
different tables, as the timestamps of a log entry is part of the key.
For example, a table written at update index 20 may contain a tombstone
log entry at timestamp 1.
Therefore, we set ReftableWriter's min/maxUpdateIndex from the merged
tables we are compacting, rather than from the compaction settings
(which should only control reflog expiry.)
The previous behavior could drop log entries erroneously, especially
in the presence of tombstone log entries. Unfortunately, testing this
properly requires both an API for adding log tombstones, and a more
refined API for controlling automatic compaction. Hence, no test.
Change-Id: I2f4eb7866f607fddd0629809e8e61f0b9097717f
Signed-off-by: Han-Wen Nienhuys <hanwen@google.com>
4 年之前 Simplify ReftableCompactor
The ReftableCompactor supported a byteLimit, but this is currently
unused. The FileReftableStack has a more sophisticated strategy that
amortizes compaction costs.
Rename min/maxUpdateIndex to reflogExpire{Min,Max}UpdateIndex to
reflect their purpose more accurately.
Since reflogs are generally pruned chronologically (oldest entries are
expired first), one can only prune entries on full compaction, so they
should not be set by default.
Rephrase the function Reader#minUpdateIndex and maxUpdateIndex. These
vars are documented to affect log entries, but semantically, they are
about ref entries. Since ref entries have their timestamps
delta-compressed, it is important for the min/maxUpdateIndex values to
be coherent between different tables.
The logical timestamps for log entries do not have to be coherent in
different tables, as the timestamps of a log entry is part of the key.
For example, a table written at update index 20 may contain a tombstone
log entry at timestamp 1.
Therefore, we set ReftableWriter's min/maxUpdateIndex from the merged
tables we are compacting, rather than from the compaction settings
(which should only control reflog expiry.)
The previous behavior could drop log entries erroneously, especially
in the presence of tombstone log entries. Unfortunately, testing this
properly requires both an API for adding log tombstones, and a more
refined API for controlling automatic compaction. Hence, no test.
Change-Id: I2f4eb7866f607fddd0629809e8e61f0b9097717f
Signed-off-by: Han-Wen Nienhuys <hanwen@google.com>
4 年之前 |
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424 |
- /*
- * Copyright (C) 2017, Google Inc. and others
- *
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Distribution License v. 1.0 which is available at
- * https://www.eclipse.org/org/documents/edl-v10.php.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
- package org.eclipse.jgit.internal.storage.reftable;
-
- import java.io.IOException;
- import java.util.List;
- import java.util.PriorityQueue;
-
- import org.eclipse.jgit.lib.AnyObjectId;
- import org.eclipse.jgit.lib.Ref;
- import org.eclipse.jgit.lib.ReflogEntry;
-
- /**
- * Merges multiple reference tables together.
- * <p>
- * A {@link org.eclipse.jgit.internal.storage.reftable.MergedReftable}
- * merge-joins multiple
- * {@link org.eclipse.jgit.internal.storage.reftable.ReftableReader} on the fly.
- * Tables higher/later in the stack shadow lower/earlier tables, hiding
- * references that been updated/replaced.
- * <p>
- * By default deleted references are skipped and not returned to the caller.
- * {@link #setIncludeDeletes(boolean)} can be used to modify this behavior if
- * the caller needs to preserve deletions during partial compaction.
- * <p>
- * A {@code MergedReftable} is not thread-safe.
- */
- public class MergedReftable extends Reftable {
- private final ReftableReader[] tables;
-
- /**
- * Initialize a merged table reader.
- * <p>
- *
- * @param tableStack
- * stack of tables to read from. The base of the stack is at
- * index 0, the most recent should be at the top of the stack at
- * {@code tableStack.size() - 1}. The top of the stack (higher
- * index) shadows the base of the stack (lower index).
- */
- public MergedReftable(List<ReftableReader> tableStack) {
- tables = tableStack.toArray(new ReftableReader[0]);
-
- // Tables must expose deletes to this instance to correctly
- // shadow references from lower tables.
- for (ReftableReader t : tables) {
- t.setIncludeDeletes(true);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public long maxUpdateIndex() throws IOException {
- if (tables.length == 0) {
- return 0;
- }
- long maxUpdateIndex = tables[tables.length - 1].maxUpdateIndex();
- for (int i = tables.length - 2; i >= 0; i--) {
- if (maxUpdateIndex < tables[i].maxUpdateIndex()) {
- maxUpdateIndex = tables[i].maxUpdateIndex();
- }
- }
- return maxUpdateIndex;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public long minUpdateIndex() throws IOException {
- if (tables.length == 0) {
- return 0;
- }
- long minUpdateIndex = tables[0].minUpdateIndex();
- for (int i = 0; i < tables.length - 1; i++) {
- if (tables[i].minUpdateIndex() < minUpdateIndex) {
- minUpdateIndex = tables[i].minUpdateIndex();
- }
- }
- return minUpdateIndex;
- }
-
- /** {@inheritDoc} */
- @Override
- public boolean hasObjectMap() throws IOException {
- boolean has = true;
- for (int i = 0; has && i < tables.length; i++) {
- has = has && tables[i].hasObjectMap();
- }
- return has;
- }
-
- /** {@inheritDoc} */
- @Override
- public RefCursor allRefs() throws IOException {
- MergedRefCursor m = new MergedRefCursor();
- for (int i = 0; i < tables.length; i++) {
- m.add(new RefQueueEntry(tables[i].allRefs(), i));
- }
- return m;
- }
-
- /** {@inheritDoc} */
- @Override
- public RefCursor seekRef(String name) throws IOException {
- MergedRefCursor m = new MergedRefCursor();
- for (int i = 0; i < tables.length; i++) {
- m.add(new RefQueueEntry(tables[i].seekRef(name), i));
- }
- return m;
- }
-
- /** {@inheritDoc} */
- @Override
- public RefCursor seekRefsWithPrefix(String prefix) throws IOException {
- MergedRefCursor m = new MergedRefCursor();
- for (int i = 0; i < tables.length; i++) {
- m.add(new RefQueueEntry(tables[i].seekRefsWithPrefix(prefix), i));
- }
- return m;
- }
-
- /** {@inheritDoc} */
- @Override
- public RefCursor byObjectId(AnyObjectId name) throws IOException {
- MergedRefCursor m = new FilteringMergedRefCursor(name);
- for (int i = 0; i < tables.length; i++) {
- m.add(new RefQueueEntry(tables[i].byObjectId(name), i));
- }
- return m;
- }
-
- /** {@inheritDoc} */
- @Override
- public LogCursor allLogs() throws IOException {
- MergedLogCursor m = new MergedLogCursor();
- for (int i = 0; i < tables.length; i++) {
- m.add(new LogQueueEntry(tables[i].allLogs(), i));
- }
- return m;
- }
-
- /** {@inheritDoc} */
- @Override
- public LogCursor seekLog(String refName, long updateIdx)
- throws IOException {
- MergedLogCursor m = new MergedLogCursor();
- for (int i = 0; i < tables.length; i++) {
- m.add(new LogQueueEntry(tables[i].seekLog(refName, updateIdx), i));
- }
- return m;
- }
-
- int queueSize() {
- return Math.max(1, tables.length);
- }
-
- private class MergedRefCursor extends RefCursor {
- private final PriorityQueue<RefQueueEntry> queue;
- private RefQueueEntry head;
- private Ref ref;
-
- MergedRefCursor() {
- queue = new PriorityQueue<>(queueSize(), RefQueueEntry::compare);
- }
-
- void add(RefQueueEntry t) throws IOException {
- // Common case is many iterations over the same RefQueueEntry
- // for the bottom of the stack (scanning all refs). Its almost
- // always less than the top of the queue. Avoid the queue's
- // O(log N) insertion and removal costs for this common case.
- if (!t.rc.next()) {
- t.rc.close();
- } else if (head == null) {
- RefQueueEntry p = queue.peek();
- if (p == null || RefQueueEntry.compare(t, p) < 0) {
- head = t;
- } else {
- head = queue.poll();
- queue.add(t);
- }
- } else if (RefQueueEntry.compare(t, head) > 0) {
- queue.add(t);
- } else {
- queue.add(head);
- head = t;
- }
- }
-
- @Override
- public boolean next() throws IOException {
- for (;;) {
- RefQueueEntry t = poll();
- if (t == null) {
- return false;
- }
-
- ref = t.rc.getRef();
- boolean include = includeDeletes || !t.rc.wasDeleted();
- add(t);
- skipShadowedRefs(ref.getName());
- if (include) {
- return true;
- }
- }
- }
-
- private RefQueueEntry poll() {
- RefQueueEntry e = head;
- if (e != null) {
- head = null;
- return e;
- }
- return queue.poll();
- }
-
- private void skipShadowedRefs(String name) throws IOException {
- for (;;) {
- RefQueueEntry t = head != null ? head : queue.peek();
- if (t != null && name.equals(t.name())) {
- add(poll());
- } else {
- break;
- }
- }
- }
-
- @Override
- public Ref getRef() {
- return ref;
- }
-
- @Override
- public void close() {
- if (head != null) {
- head.rc.close();
- head = null;
- }
- while (!queue.isEmpty()) {
- queue.remove().rc.close();
- }
- }
- }
-
- private class FilteringMergedRefCursor extends MergedRefCursor {
- final AnyObjectId filterId;
- Ref filteredRef;
-
- FilteringMergedRefCursor(AnyObjectId id) {
- filterId = id;
- filteredRef = null;
- }
-
- @Override
- public Ref getRef() {
- return filteredRef;
- }
-
- @Override
- public boolean next() throws IOException {
- for (;;) {
- boolean ok = super.next();
- if (!ok) {
- return false;
- }
-
- String name = super.getRef().getName();
-
- try (RefCursor c = seekRef(name)) {
- if (c.next()) {
- if (filterId.equals(c.getRef().getObjectId())) {
- filteredRef = c.getRef();
- return true;
- }
- }
- }
- }
- }
- }
-
- private static class RefQueueEntry {
- static int compare(RefQueueEntry a, RefQueueEntry b) {
- int cmp = a.name().compareTo(b.name());
- if (cmp == 0) {
- // higher updateIndex shadows lower updateIndex.
- cmp = Long.signum(b.updateIndex() - a.updateIndex());
- }
- if (cmp == 0) {
- // higher index shadows lower index, so higher index first.
- cmp = b.stackIdx - a.stackIdx;
- }
- return cmp;
- }
-
- final RefCursor rc;
- final int stackIdx;
-
- RefQueueEntry(RefCursor rc, int stackIdx) {
- this.rc = rc;
- this.stackIdx = stackIdx;
- }
-
- String name() {
- return rc.getRef().getName();
- }
-
- long updateIndex() {
- return rc.getRef().getUpdateIndex();
- }
- }
-
- private class MergedLogCursor extends LogCursor {
- private final PriorityQueue<LogQueueEntry> queue;
- private String refName;
- private long updateIndex;
- private ReflogEntry entry;
-
- MergedLogCursor() {
- queue = new PriorityQueue<>(queueSize(), LogQueueEntry::compare);
- }
-
- void add(LogQueueEntry t) throws IOException {
- if (t.lc.next()) {
- queue.add(t);
- } else {
- t.lc.close();
- }
- }
-
- @Override
- public boolean next() throws IOException {
- for (;;) {
- LogQueueEntry t = queue.poll();
- if (t == null) {
- return false;
- }
-
- refName = t.lc.getRefName();
- updateIndex = t.lc.getUpdateIndex();
- entry = t.lc.getReflogEntry();
- boolean include = includeDeletes || entry != null;
- skipShadowed(refName, updateIndex);
- add(t);
- if (include) {
- return true;
- }
- }
- }
-
- private void skipShadowed(String name, long index) throws IOException {
- for (;;) {
- LogQueueEntry t = queue.peek();
- if (t != null && name.equals(t.name()) && index == t.index()) {
- add(queue.remove());
- } else {
- break;
- }
- }
- }
-
- @Override
- public String getRefName() {
- return refName;
- }
-
- @Override
- public long getUpdateIndex() {
- return updateIndex;
- }
-
- @Override
- public ReflogEntry getReflogEntry() {
- return entry;
- }
-
- @Override
- public void close() {
- while (!queue.isEmpty()) {
- queue.remove().lc.close();
- }
- }
- }
-
- private static class LogQueueEntry {
- static int compare(LogQueueEntry a, LogQueueEntry b) {
- int cmp = a.name().compareTo(b.name());
- if (cmp == 0) {
- // higher update index sorts first.
- cmp = Long.signum(b.index() - a.index());
- }
- if (cmp == 0) {
- // higher index comes first.
- cmp = b.stackIdx - a.stackIdx;
- }
- return cmp;
- }
-
- final LogCursor lc;
- final int stackIdx;
-
- LogQueueEntry(LogCursor lc, int stackIdx) {
- this.lc = lc;
- this.stackIdx = stackIdx;
- }
-
- String name() {
- return lc.getRefName();
- }
-
- long index() {
- return lc.getUpdateIndex();
- }
- }
- }
|