You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

MergedReftable.java 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432
  1. /*
  2. * Copyright (C) 2017, Google Inc.
  3. * and other copyright owners as documented in the project's IP log.
  4. *
  5. * This program and the accompanying materials are made available
  6. * under the terms of the Eclipse Distribution License v1.0 which
  7. * accompanies this distribution, is reproduced below, and is
  8. * available at http://www.eclipse.org/org/documents/edl-v10.php
  9. *
  10. * All rights reserved.
  11. *
  12. * Redistribution and use in source and binary forms, with or
  13. * without modification, are permitted provided that the following
  14. * conditions are met:
  15. *
  16. * - Redistributions of source code must retain the above copyright
  17. * notice, this list of conditions and the following disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials provided
  22. * with the distribution.
  23. *
  24. * - Neither the name of the Eclipse Foundation, Inc. nor the
  25. * names of its contributors may be used to endorse or promote
  26. * products derived from this software without specific prior
  27. * written permission.
  28. *
  29. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  30. * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  31. * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  32. * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  34. * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  35. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  36. * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  37. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  38. * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  39. * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  40. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  41. * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42. */
  43. package org.eclipse.jgit.internal.storage.reftable;
  44. import java.io.IOException;
  45. import java.util.List;
  46. import java.util.PriorityQueue;
  47. import org.eclipse.jgit.lib.AnyObjectId;
  48. import org.eclipse.jgit.lib.Ref;
  49. import org.eclipse.jgit.lib.ReflogEntry;
  50. /**
  51. * Merges multiple reference tables together.
  52. * <p>
  53. * A {@link org.eclipse.jgit.internal.storage.reftable.MergedReftable}
  54. * merge-joins multiple
  55. * {@link org.eclipse.jgit.internal.storage.reftable.ReftableReader} on the fly.
  56. * Tables higher/later in the stack shadow lower/earlier tables, hiding
  57. * references that been updated/replaced.
  58. * <p>
  59. * By default deleted references are skipped and not returned to the caller.
  60. * {@link #setIncludeDeletes(boolean)} can be used to modify this behavior if
  61. * the caller needs to preserve deletions during partial compaction.
  62. * <p>
  63. * A {@code MergedReftable} is not thread-safe.
  64. */
  65. public class MergedReftable extends Reftable {
  66. private final ReftableReader[] tables;
  67. /**
  68. * Initialize a merged table reader.
  69. * <p>
  70. *
  71. * @param tableStack
  72. * stack of tables to read from. The base of the stack is at
  73. * index 0, the most recent should be at the top of the stack at
  74. * {@code tableStack.size() - 1}. The top of the stack (higher
  75. * index) shadows the base of the stack (lower index).
  76. */
  77. public MergedReftable(List<ReftableReader> tableStack) {
  78. tables = tableStack.toArray(new ReftableReader[0]);
  79. // Tables must expose deletes to this instance to correctly
  80. // shadow references from lower tables.
  81. for (ReftableReader t : tables) {
  82. t.setIncludeDeletes(true);
  83. }
  84. }
  85. /**
  86. * {@inheritDoc}
  87. */
  88. @Override
  89. public long maxUpdateIndex() throws IOException {
  90. return tables.length > 0 ? tables[tables.length - 1].maxUpdateIndex()
  91. : 0;
  92. }
  93. /** {@inheritDoc} */
  94. @Override
  95. public boolean hasObjectMap() throws IOException {
  96. boolean has = true;
  97. for (int i = 0; has && i < tables.length; i++) {
  98. has = has && tables[i].hasObjectMap();
  99. };
  100. return has;
  101. }
  102. /** {@inheritDoc} */
  103. @Override
  104. public RefCursor allRefs() throws IOException {
  105. MergedRefCursor m = new MergedRefCursor();
  106. for (int i = 0; i < tables.length; i++) {
  107. m.add(new RefQueueEntry(tables[i].allRefs(), i));
  108. }
  109. return m;
  110. }
  111. /** {@inheritDoc} */
  112. @Override
  113. public RefCursor seekRef(String name) throws IOException {
  114. MergedRefCursor m = new MergedRefCursor();
  115. for (int i = 0; i < tables.length; i++) {
  116. m.add(new RefQueueEntry(tables[i].seekRef(name), i));
  117. }
  118. return m;
  119. }
  120. /** {@inheritDoc} */
  121. @Override
  122. public RefCursor seekRefsWithPrefix(String prefix) throws IOException {
  123. MergedRefCursor m = new MergedRefCursor();
  124. for (int i = 0; i < tables.length; i++) {
  125. m.add(new RefQueueEntry(tables[i].seekRefsWithPrefix(prefix), i));
  126. }
  127. return m;
  128. }
  129. /** {@inheritDoc} */
  130. @Override
  131. public RefCursor byObjectId(AnyObjectId name) throws IOException {
  132. MergedRefCursor m = new FilteringMergedRefCursor(name);
  133. for (int i = 0; i < tables.length; i++) {
  134. m.add(new RefQueueEntry(tables[i].byObjectId(name), i));
  135. }
  136. return m;
  137. }
  138. /** {@inheritDoc} */
  139. @Override
  140. public LogCursor allLogs() throws IOException {
  141. MergedLogCursor m = new MergedLogCursor();
  142. for (int i = 0; i < tables.length; i++) {
  143. m.add(new LogQueueEntry(tables[i].allLogs(), i));
  144. }
  145. return m;
  146. }
  147. /** {@inheritDoc} */
  148. @Override
  149. public LogCursor seekLog(String refName, long updateIdx)
  150. throws IOException {
  151. MergedLogCursor m = new MergedLogCursor();
  152. for (int i = 0; i < tables.length; i++) {
  153. m.add(new LogQueueEntry(tables[i].seekLog(refName, updateIdx), i));
  154. }
  155. return m;
  156. }
  157. int queueSize() {
  158. return Math.max(1, tables.length);
  159. }
  160. private class MergedRefCursor extends RefCursor {
  161. private final PriorityQueue<RefQueueEntry> queue;
  162. private RefQueueEntry head;
  163. private Ref ref;
  164. MergedRefCursor() {
  165. queue = new PriorityQueue<>(queueSize(), RefQueueEntry::compare);
  166. }
  167. void add(RefQueueEntry t) throws IOException {
  168. // Common case is many iterations over the same RefQueueEntry
  169. // for the bottom of the stack (scanning all refs). Its almost
  170. // always less than the top of the queue. Avoid the queue's
  171. // O(log N) insertion and removal costs for this common case.
  172. if (!t.rc.next()) {
  173. t.rc.close();
  174. } else if (head == null) {
  175. RefQueueEntry p = queue.peek();
  176. if (p == null || RefQueueEntry.compare(t, p) < 0) {
  177. head = t;
  178. } else {
  179. head = queue.poll();
  180. queue.add(t);
  181. }
  182. } else if (RefQueueEntry.compare(t, head) > 0) {
  183. queue.add(t);
  184. } else {
  185. queue.add(head);
  186. head = t;
  187. }
  188. }
  189. @Override
  190. public boolean next() throws IOException {
  191. for (;;) {
  192. RefQueueEntry t = poll();
  193. if (t == null) {
  194. return false;
  195. }
  196. ref = t.rc.getRef();
  197. boolean include = includeDeletes || !t.rc.wasDeleted();
  198. add(t);
  199. skipShadowedRefs(ref.getName());
  200. if (include) {
  201. return true;
  202. }
  203. }
  204. }
  205. private RefQueueEntry poll() {
  206. RefQueueEntry e = head;
  207. if (e != null) {
  208. head = null;
  209. return e;
  210. }
  211. return queue.poll();
  212. }
  213. private void skipShadowedRefs(String name) throws IOException {
  214. for (;;) {
  215. RefQueueEntry t = head != null ? head : queue.peek();
  216. if (t != null && name.equals(t.name())) {
  217. add(poll());
  218. } else {
  219. break;
  220. }
  221. }
  222. }
  223. @Override
  224. public Ref getRef() {
  225. return ref;
  226. }
  227. @Override
  228. public void close() {
  229. if (head != null) {
  230. head.rc.close();
  231. head = null;
  232. }
  233. while (!queue.isEmpty()) {
  234. queue.remove().rc.close();
  235. }
  236. }
  237. }
  238. private class FilteringMergedRefCursor extends MergedRefCursor {
  239. final AnyObjectId filterId;
  240. Ref filteredRef;
  241. FilteringMergedRefCursor(AnyObjectId id) {
  242. filterId = id;
  243. filteredRef = null;
  244. }
  245. @Override
  246. public Ref getRef() {
  247. return filteredRef;
  248. }
  249. @Override
  250. public boolean next() throws IOException {
  251. for (;;) {
  252. boolean ok = super.next();
  253. if (!ok) {
  254. return false;
  255. }
  256. String name = super.getRef().getName();
  257. try (RefCursor c = seekRef(name)) {
  258. if (c.next()) {
  259. if (filterId.equals(c.getRef().getObjectId())) {
  260. filteredRef = c.getRef();
  261. return true;
  262. }
  263. }
  264. }
  265. }
  266. }
  267. }
  268. private static class RefQueueEntry {
  269. static int compare(RefQueueEntry a, RefQueueEntry b) {
  270. int cmp = a.name().compareTo(b.name());
  271. if (cmp == 0) {
  272. // higher updateIndex shadows lower updateIndex.
  273. cmp = Long.signum(b.updateIndex() - a.updateIndex());
  274. }
  275. if (cmp == 0) {
  276. // higher index shadows lower index, so higher index first.
  277. cmp = b.stackIdx - a.stackIdx;
  278. }
  279. return cmp;
  280. }
  281. final RefCursor rc;
  282. final int stackIdx;
  283. RefQueueEntry(RefCursor rc, int stackIdx) {
  284. this.rc = rc;
  285. this.stackIdx = stackIdx;
  286. }
  287. String name() {
  288. return rc.getRef().getName();
  289. }
  290. long updateIndex() {
  291. return rc.getRef().getUpdateIndex();
  292. }
  293. }
  294. private class MergedLogCursor extends LogCursor {
  295. private final PriorityQueue<LogQueueEntry> queue;
  296. private String refName;
  297. private long updateIndex;
  298. private ReflogEntry entry;
  299. MergedLogCursor() {
  300. queue = new PriorityQueue<>(queueSize(), LogQueueEntry::compare);
  301. }
  302. void add(LogQueueEntry t) throws IOException {
  303. if (t.lc.next()) {
  304. queue.add(t);
  305. } else {
  306. t.lc.close();
  307. }
  308. }
  309. @Override
  310. public boolean next() throws IOException {
  311. for (;;) {
  312. LogQueueEntry t = queue.poll();
  313. if (t == null) {
  314. return false;
  315. }
  316. refName = t.lc.getRefName();
  317. updateIndex = t.lc.getUpdateIndex();
  318. entry = t.lc.getReflogEntry();
  319. boolean include = includeDeletes || entry != null;
  320. skipShadowed(refName, updateIndex);
  321. add(t);
  322. if (include) {
  323. return true;
  324. }
  325. }
  326. }
  327. private void skipShadowed(String name, long index) throws IOException {
  328. for (;;) {
  329. LogQueueEntry t = queue.peek();
  330. if (t != null && name.equals(t.name()) && index == t.index()) {
  331. add(queue.remove());
  332. } else {
  333. break;
  334. }
  335. }
  336. }
  337. @Override
  338. public String getRefName() {
  339. return refName;
  340. }
  341. @Override
  342. public long getUpdateIndex() {
  343. return updateIndex;
  344. }
  345. @Override
  346. public ReflogEntry getReflogEntry() {
  347. return entry;
  348. }
  349. @Override
  350. public void close() {
  351. while (!queue.isEmpty()) {
  352. queue.remove().lc.close();
  353. }
  354. }
  355. }
  356. private static class LogQueueEntry {
  357. static int compare(LogQueueEntry a, LogQueueEntry b) {
  358. int cmp = a.name().compareTo(b.name());
  359. if (cmp == 0) {
  360. // higher update index sorts first.
  361. cmp = Long.signum(b.index() - a.index());
  362. }
  363. if (cmp == 0) {
  364. // higher index comes first.
  365. cmp = b.stackIdx - a.stackIdx;
  366. }
  367. return cmp;
  368. }
  369. final LogCursor lc;
  370. final int stackIdx;
  371. LogQueueEntry(LogCursor lc, int stackIdx) {
  372. this.lc = lc;
  373. this.stackIdx = stackIdx;
  374. }
  375. String name() {
  376. return lc.getRefName();
  377. }
  378. long index() {
  379. return lc.getUpdateIndex();
  380. }
  381. }
  382. }