You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

WorkingTreeIterator.java 48KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593
  1. /*
  2. * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
  3. * Copyright (C) 2010, Christian Halstrick <christian.halstrick@sap.com>
  4. * Copyright (C) 2010, Matthias Sohn <matthias.sohn@sap.com>
  5. * Copyright (C) 2012-2021, Robin Rosenberg and others
  6. *
  7. * This program and the accompanying materials are made available under the
  8. * terms of the Eclipse Distribution License v. 1.0 which is available at
  9. * https://www.eclipse.org/org/documents/edl-v10.php.
  10. *
  11. * SPDX-License-Identifier: BSD-3-Clause
  12. */
  13. package org.eclipse.jgit.treewalk;
  14. import static java.nio.charset.StandardCharsets.UTF_8;
  15. import java.io.ByteArrayInputStream;
  16. import java.io.File;
  17. import java.io.FileInputStream;
  18. import java.io.FileNotFoundException;
  19. import java.io.IOException;
  20. import java.io.InputStream;
  21. import java.nio.ByteBuffer;
  22. import java.nio.CharBuffer;
  23. import java.nio.charset.CharacterCodingException;
  24. import java.nio.charset.CharsetEncoder;
  25. import java.nio.file.Path;
  26. import java.text.MessageFormat;
  27. import java.time.Instant;
  28. import java.util.Arrays;
  29. import java.util.Collections;
  30. import java.util.Comparator;
  31. import java.util.HashMap;
  32. import java.util.Map;
  33. import org.eclipse.jgit.api.errors.FilterFailedException;
  34. import org.eclipse.jgit.attributes.AttributesNode;
  35. import org.eclipse.jgit.attributes.AttributesRule;
  36. import org.eclipse.jgit.attributes.FilterCommand;
  37. import org.eclipse.jgit.attributes.FilterCommandRegistry;
  38. import org.eclipse.jgit.diff.RawText;
  39. import org.eclipse.jgit.dircache.DirCacheEntry;
  40. import org.eclipse.jgit.dircache.DirCacheIterator;
  41. import org.eclipse.jgit.errors.CorruptObjectException;
  42. import org.eclipse.jgit.errors.LargeObjectException;
  43. import org.eclipse.jgit.errors.MissingObjectException;
  44. import org.eclipse.jgit.errors.NoWorkTreeException;
  45. import org.eclipse.jgit.ignore.FastIgnoreRule;
  46. import org.eclipse.jgit.ignore.IgnoreNode;
  47. import org.eclipse.jgit.internal.JGitText;
  48. import org.eclipse.jgit.lib.ConfigConstants;
  49. import org.eclipse.jgit.lib.Constants;
  50. import org.eclipse.jgit.lib.CoreConfig.CheckStat;
  51. import org.eclipse.jgit.lib.CoreConfig.EolStreamType;
  52. import org.eclipse.jgit.lib.CoreConfig.SymLinks;
  53. import org.eclipse.jgit.lib.FileMode;
  54. import org.eclipse.jgit.lib.ObjectId;
  55. import org.eclipse.jgit.lib.ObjectLoader;
  56. import org.eclipse.jgit.lib.ObjectReader;
  57. import org.eclipse.jgit.lib.Repository;
  58. import org.eclipse.jgit.submodule.SubmoduleWalk;
  59. import org.eclipse.jgit.treewalk.TreeWalk.OperationType;
  60. import org.eclipse.jgit.util.FS;
  61. import org.eclipse.jgit.util.FS.ExecutionResult;
  62. import org.eclipse.jgit.util.FileUtils;
  63. import org.eclipse.jgit.util.Holder;
  64. import org.eclipse.jgit.util.IO;
  65. import org.eclipse.jgit.util.Paths;
  66. import org.eclipse.jgit.util.RawParseUtils;
  67. import org.eclipse.jgit.util.TemporaryBuffer;
  68. import org.eclipse.jgit.util.TemporaryBuffer.LocalFile;
  69. import org.eclipse.jgit.util.io.EolStreamTypeUtil;
  70. import org.eclipse.jgit.util.sha1.SHA1;
  71. /**
  72. * Walks a working directory tree as part of a
  73. * {@link org.eclipse.jgit.treewalk.TreeWalk}.
  74. * <p>
  75. * Most applications will want to use the standard implementation of this
  76. * iterator, {@link org.eclipse.jgit.treewalk.FileTreeIterator}, as that does
  77. * all IO through the standard <code>java.io</code> package. Plugins for a Java
  78. * based IDE may however wish to create their own implementations of this class
  79. * to allow traversal of the IDE's project space, as well as benefit from any
  80. * caching the IDE may have.
  81. *
  82. * @see FileTreeIterator
  83. */
  84. public abstract class WorkingTreeIterator extends AbstractTreeIterator {
  85. private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024;
  86. /** An empty entry array, suitable for {@link #init(Entry[])}. */
  87. protected static final Entry[] EOF = {};
  88. /** Size we perform file IO in if we have to read and hash a file. */
  89. static final int BUFFER_SIZE = 2048;
  90. /**
  91. * Maximum size of files which may be read fully into memory for performance
  92. * reasons.
  93. */
  94. private static final long MAXIMUM_FILE_SIZE_TO_READ_FULLY = 65536;
  95. /** Inherited state of this iterator, describing working tree, etc. */
  96. private final IteratorState state;
  97. /** The {@link #idBuffer()} for the current entry. */
  98. private byte[] contentId;
  99. /** Index within {@link #entries} that {@link #contentId} came from. */
  100. private int contentIdFromPtr;
  101. /** List of entries obtained from the subclass. */
  102. private Entry[] entries;
  103. /** Total number of entries in {@link #entries} that are valid. */
  104. private int entryCnt;
  105. /** Current position within {@link #entries}. */
  106. private int ptr;
  107. /** If there is a .gitignore file present, the parsed rules from it. */
  108. private IgnoreNode ignoreNode;
  109. /**
  110. * cached clean filter command. Use a Ref in order to distinguish between
  111. * the ref not cached yet and the value null
  112. */
  113. private Holder<String> cleanFilterCommandHolder;
  114. /**
  115. * cached eol stream type. Use a Ref in order to distinguish between the ref
  116. * not cached yet and the value null
  117. */
  118. private Holder<EolStreamType> eolStreamTypeHolder;
  119. /** Repository that is the root level being iterated over */
  120. protected Repository repository;
  121. /** Cached canonical length, initialized from {@link #idBuffer()} */
  122. private long canonLen = -1;
  123. /** The offset of the content id in {@link #idBuffer()} */
  124. private int contentIdOffset;
  125. /** A comparator for {@link Instant}s. */
  126. private final InstantComparator timestampComparator = new InstantComparator();
  127. /**
  128. * Create a new iterator with no parent.
  129. *
  130. * @param options
  131. * working tree options to be used
  132. */
  133. protected WorkingTreeIterator(WorkingTreeOptions options) {
  134. super();
  135. state = new IteratorState(options);
  136. }
  137. /**
  138. * Create a new iterator with no parent and a prefix.
  139. * <p>
  140. * The prefix path supplied is inserted in front of all paths generated by
  141. * this iterator. It is intended to be used when an iterator is being
  142. * created for a subsection of an overall repository and needs to be
  143. * combined with other iterators that are created to run over the entire
  144. * repository namespace.
  145. *
  146. * @param prefix
  147. * position of this iterator in the repository tree. The value
  148. * may be null or the empty string to indicate the prefix is the
  149. * root of the repository. A trailing slash ('/') is
  150. * automatically appended if the prefix does not end in '/'.
  151. * @param options
  152. * working tree options to be used
  153. */
  154. protected WorkingTreeIterator(final String prefix,
  155. WorkingTreeOptions options) {
  156. super(prefix);
  157. state = new IteratorState(options);
  158. }
  159. /**
  160. * Create an iterator for a subtree of an existing iterator.
  161. *
  162. * @param p
  163. * parent tree iterator.
  164. */
  165. protected WorkingTreeIterator(WorkingTreeIterator p) {
  166. super(p);
  167. state = p.state;
  168. repository = p.repository;
  169. }
  170. /**
  171. * Initialize this iterator for the root level of a repository.
  172. * <p>
  173. * This method should only be invoked after calling {@link #init(Entry[])},
  174. * and only for the root iterator.
  175. *
  176. * @param repo
  177. * the repository.
  178. */
  179. protected void initRootIterator(Repository repo) {
  180. repository = repo;
  181. Entry entry;
  182. if (ignoreNode instanceof PerDirectoryIgnoreNode)
  183. entry = ((PerDirectoryIgnoreNode) ignoreNode).entry;
  184. else
  185. entry = null;
  186. ignoreNode = new RootIgnoreNode(entry, repo);
  187. }
  188. /**
  189. * Define the matching {@link org.eclipse.jgit.dircache.DirCacheIterator},
  190. * to optimize ObjectIds.
  191. *
  192. * Once the DirCacheIterator has been set this iterator must only be
  193. * advanced by the TreeWalk that is supplied, as it assumes that itself and
  194. * the corresponding DirCacheIterator are positioned on the same file path
  195. * whenever {@link #idBuffer()} is invoked.
  196. *
  197. * @param walk
  198. * the walk that will be advancing this iterator.
  199. * @param treeId
  200. * index of the matching
  201. * {@link org.eclipse.jgit.dircache.DirCacheIterator}.
  202. */
  203. public void setDirCacheIterator(TreeWalk walk, int treeId) {
  204. state.walk = walk;
  205. state.dirCacheTree = treeId;
  206. }
  207. /**
  208. * Retrieves the {@link DirCacheIterator} at the current entry if
  209. * {@link #setDirCacheIterator(TreeWalk, int)} was called.
  210. *
  211. * @return the DirCacheIterator, or {@code null} if not set or not at the
  212. * current entry
  213. * @since 5.0
  214. */
  215. protected DirCacheIterator getDirCacheIterator() {
  216. if (state.dirCacheTree >= 0 && state.walk != null) {
  217. return state.walk.getTree(state.dirCacheTree,
  218. DirCacheIterator.class);
  219. }
  220. return null;
  221. }
  222. /**
  223. * Defines whether this {@link WorkingTreeIterator} walks ignored
  224. * directories.
  225. *
  226. * @param includeIgnored
  227. * {@code false} to skip ignored directories, if possible;
  228. * {@code true} to always include them in the walk
  229. * @since 5.0
  230. */
  231. public void setWalkIgnoredDirectories(boolean includeIgnored) {
  232. state.walkIgnored = includeIgnored;
  233. }
  234. /**
  235. * Tells whether this {@link WorkingTreeIterator} walks ignored directories.
  236. *
  237. * @return {@code true} if it does, {@code false} otherwise
  238. * @since 5.0
  239. */
  240. public boolean walksIgnoredDirectories() {
  241. return state.walkIgnored;
  242. }
  243. /** {@inheritDoc} */
  244. @Override
  245. public boolean hasId() {
  246. if (contentIdFromPtr == ptr)
  247. return true;
  248. return (mode & FileMode.TYPE_MASK) == FileMode.TYPE_FILE;
  249. }
  250. /** {@inheritDoc} */
  251. @Override
  252. public byte[] idBuffer() {
  253. if (contentIdFromPtr == ptr)
  254. return contentId;
  255. if (state.walk != null) {
  256. // If there is a matching DirCacheIterator, we can reuse
  257. // its idBuffer, but only if we appear to be clean against
  258. // the cached index information for the path.
  259. DirCacheIterator i = state.walk.getTree(state.dirCacheTree,
  260. DirCacheIterator.class);
  261. if (i != null) {
  262. DirCacheEntry ent = i.getDirCacheEntry();
  263. if (ent != null && compareMetadata(ent) == MetadataDiff.EQUAL
  264. && ((ent.getFileMode().getBits()
  265. & FileMode.TYPE_MASK) != FileMode.TYPE_GITLINK)) {
  266. contentIdOffset = i.idOffset();
  267. contentIdFromPtr = ptr;
  268. return contentId = i.idBuffer();
  269. }
  270. contentIdOffset = 0;
  271. } else {
  272. contentIdOffset = 0;
  273. }
  274. }
  275. switch (mode & FileMode.TYPE_MASK) {
  276. case FileMode.TYPE_SYMLINK:
  277. case FileMode.TYPE_FILE:
  278. contentIdFromPtr = ptr;
  279. return contentId = idBufferBlob(entries[ptr]);
  280. case FileMode.TYPE_GITLINK:
  281. contentIdFromPtr = ptr;
  282. return contentId = idSubmodule(entries[ptr]);
  283. }
  284. return zeroid;
  285. }
  286. /** {@inheritDoc} */
  287. @Override
  288. public boolean isWorkTree() {
  289. return true;
  290. }
  291. /**
  292. * Get submodule id for given entry.
  293. *
  294. * @param e
  295. * a {@link org.eclipse.jgit.treewalk.WorkingTreeIterator.Entry}
  296. * object.
  297. * @return non-null submodule id
  298. */
  299. protected byte[] idSubmodule(Entry e) {
  300. if (repository == null)
  301. return zeroid;
  302. File directory;
  303. try {
  304. directory = repository.getWorkTree();
  305. } catch (NoWorkTreeException nwte) {
  306. return zeroid;
  307. }
  308. return idSubmodule(directory, e);
  309. }
  310. /**
  311. * Get submodule id using the repository at the location of the entry
  312. * relative to the directory.
  313. *
  314. * @param directory
  315. * a {@link java.io.File} object.
  316. * @param e
  317. * a {@link org.eclipse.jgit.treewalk.WorkingTreeIterator.Entry}
  318. * object.
  319. * @return non-null submodule id
  320. */
  321. protected byte[] idSubmodule(File directory, Entry e) {
  322. try (Repository submoduleRepo = SubmoduleWalk.getSubmoduleRepository(
  323. directory, e.getName(),
  324. repository != null ? repository.getFS() : FS.DETECTED)) {
  325. if (submoduleRepo == null) {
  326. return zeroid;
  327. }
  328. ObjectId head = submoduleRepo.resolve(Constants.HEAD);
  329. if (head == null) {
  330. return zeroid;
  331. }
  332. byte[] id = new byte[Constants.OBJECT_ID_LENGTH];
  333. head.copyRawTo(id, 0);
  334. return id;
  335. } catch (IOException exception) {
  336. return zeroid;
  337. }
  338. }
  339. private static final byte[] digits = { '0', '1', '2', '3', '4', '5', '6',
  340. '7', '8', '9' };
  341. private static final byte[] hblob = Constants
  342. .encodedTypeString(Constants.OBJ_BLOB);
  343. private byte[] idBufferBlob(Entry e) {
  344. try {
  345. final InputStream is = e.openInputStream();
  346. if (is == null)
  347. return zeroid;
  348. try {
  349. state.initializeReadBuffer();
  350. final long len = e.getLength();
  351. InputStream filteredIs = possiblyFilteredInputStream(e, is, len,
  352. OperationType.CHECKIN_OP);
  353. return computeHash(filteredIs, canonLen);
  354. } finally {
  355. safeClose(is);
  356. }
  357. } catch (IOException err) {
  358. // Can't read the file? Don't report the failure either.
  359. return zeroid;
  360. }
  361. }
  362. private InputStream possiblyFilteredInputStream(final Entry e,
  363. final InputStream is, final long len) throws IOException {
  364. return possiblyFilteredInputStream(e, is, len, null);
  365. }
  366. private InputStream possiblyFilteredInputStream(final Entry e,
  367. final InputStream is, final long len, OperationType opType)
  368. throws IOException {
  369. if (getCleanFilterCommand() == null
  370. && getEolStreamType(opType) == EolStreamType.DIRECT) {
  371. canonLen = len;
  372. return is;
  373. }
  374. if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) {
  375. ByteBuffer rawbuf = IO.readWholeStream(is, (int) len);
  376. rawbuf = filterClean(rawbuf.array(), rawbuf.limit(), opType);
  377. canonLen = rawbuf.limit();
  378. return new ByteArrayInputStream(rawbuf.array(), 0, (int) canonLen);
  379. }
  380. if (getCleanFilterCommand() == null && isBinary(e)) {
  381. canonLen = len;
  382. return is;
  383. }
  384. final InputStream lenIs = filterClean(e.openInputStream(),
  385. opType);
  386. try {
  387. canonLen = computeLength(lenIs);
  388. } finally {
  389. safeClose(lenIs);
  390. }
  391. return filterClean(is, opType);
  392. }
  393. private static void safeClose(InputStream in) {
  394. try {
  395. in.close();
  396. } catch (IOException err2) {
  397. // Suppress any error related to closing an input
  398. // stream. We don't care, we should not have any
  399. // outstanding data to flush or anything like that.
  400. }
  401. }
  402. private static boolean isBinary(Entry entry) throws IOException {
  403. InputStream in = entry.openInputStream();
  404. try {
  405. return RawText.isBinary(in);
  406. } finally {
  407. safeClose(in);
  408. }
  409. }
  410. private ByteBuffer filterClean(byte[] src, int n, OperationType opType)
  411. throws IOException {
  412. InputStream in = new ByteArrayInputStream(src);
  413. try {
  414. return IO.readWholeStream(filterClean(in, opType), n);
  415. } finally {
  416. safeClose(in);
  417. }
  418. }
  419. private InputStream filterClean(InputStream in) throws IOException {
  420. return filterClean(in, null);
  421. }
  422. private InputStream filterClean(InputStream in, OperationType opType)
  423. throws IOException {
  424. in = handleAutoCRLF(in, opType);
  425. String filterCommand = getCleanFilterCommand();
  426. if (filterCommand != null) {
  427. if (FilterCommandRegistry.isRegistered(filterCommand)) {
  428. LocalFile buffer = new TemporaryBuffer.LocalFile(null);
  429. FilterCommand command = FilterCommandRegistry
  430. .createFilterCommand(filterCommand, repository, in,
  431. buffer);
  432. while (command.run() != -1) {
  433. // loop as long as command.run() tells there is work to do
  434. }
  435. return buffer.openInputStreamWithAutoDestroy();
  436. }
  437. FS fs = repository.getFS();
  438. ProcessBuilder filterProcessBuilder = fs.runInShell(filterCommand,
  439. new String[0]);
  440. filterProcessBuilder.directory(repository.getWorkTree());
  441. filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY,
  442. repository.getDirectory().getAbsolutePath());
  443. ExecutionResult result;
  444. try {
  445. result = fs.execute(filterProcessBuilder, in);
  446. } catch (IOException | InterruptedException e) {
  447. throw new IOException(new FilterFailedException(e,
  448. filterCommand, getEntryPathString()));
  449. }
  450. int rc = result.getRc();
  451. if (rc != 0) {
  452. throw new IOException(new FilterFailedException(rc,
  453. filterCommand, getEntryPathString(),
  454. result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE),
  455. RawParseUtils.decode(result.getStderr()
  456. .toByteArray(MAX_EXCEPTION_TEXT_SIZE))));
  457. }
  458. return result.getStdout().openInputStreamWithAutoDestroy();
  459. }
  460. return in;
  461. }
  462. private InputStream handleAutoCRLF(InputStream in, OperationType opType)
  463. throws IOException {
  464. return EolStreamTypeUtil.wrapInputStream(in, getEolStreamType(opType));
  465. }
  466. /**
  467. * Returns the working tree options used by this iterator.
  468. *
  469. * @return working tree options
  470. */
  471. public WorkingTreeOptions getOptions() {
  472. return state.options;
  473. }
  474. /**
  475. * Retrieves the {@link Repository} this {@link WorkingTreeIterator}
  476. * operates on.
  477. *
  478. * @return the {@link Repository}
  479. * @since 5.9
  480. */
  481. public Repository getRepository() {
  482. return repository;
  483. }
  484. /** {@inheritDoc} */
  485. @Override
  486. public int idOffset() {
  487. return contentIdOffset;
  488. }
  489. /** {@inheritDoc} */
  490. @Override
  491. public void reset() {
  492. if (!first()) {
  493. ptr = 0;
  494. if (!eof())
  495. parseEntry();
  496. }
  497. }
  498. /** {@inheritDoc} */
  499. @Override
  500. public boolean first() {
  501. return ptr == 0;
  502. }
  503. /** {@inheritDoc} */
  504. @Override
  505. public boolean eof() {
  506. return ptr == entryCnt;
  507. }
  508. /** {@inheritDoc} */
  509. @Override
  510. public void next(int delta) throws CorruptObjectException {
  511. ptr += delta;
  512. if (!eof()) {
  513. parseEntry();
  514. }
  515. }
  516. /** {@inheritDoc} */
  517. @Override
  518. public void back(int delta) throws CorruptObjectException {
  519. ptr -= delta;
  520. parseEntry();
  521. }
  522. private void parseEntry() {
  523. final Entry e = entries[ptr];
  524. mode = e.getMode().getBits();
  525. final int nameLen = e.encodedNameLen;
  526. ensurePathCapacity(pathOffset + nameLen, pathOffset);
  527. System.arraycopy(e.encodedName, 0, path, pathOffset, nameLen);
  528. pathLen = pathOffset + nameLen;
  529. canonLen = -1;
  530. cleanFilterCommandHolder = null;
  531. eolStreamTypeHolder = null;
  532. }
  533. /**
  534. * Get the raw byte length of this entry.
  535. *
  536. * @return size of this file, in bytes.
  537. */
  538. public long getEntryLength() {
  539. return current().getLength();
  540. }
  541. /**
  542. * Get the filtered input length of this entry
  543. *
  544. * @return size of the content, in bytes
  545. * @throws java.io.IOException
  546. */
  547. public long getEntryContentLength() throws IOException {
  548. if (canonLen == -1) {
  549. long rawLen = getEntryLength();
  550. if (rawLen == 0)
  551. canonLen = 0;
  552. InputStream is = current().openInputStream();
  553. try {
  554. // canonLen gets updated here
  555. possiblyFilteredInputStream(current(), is, current()
  556. .getLength());
  557. } finally {
  558. safeClose(is);
  559. }
  560. }
  561. return canonLen;
  562. }
  563. /**
  564. * Get the last modified time of this entry.
  565. *
  566. * @return last modified time of this file, in milliseconds since the epoch
  567. * (Jan 1, 1970 UTC).
  568. * @deprecated use {@link #getEntryLastModifiedInstant()} instead
  569. */
  570. @Deprecated
  571. public long getEntryLastModified() {
  572. return current().getLastModified();
  573. }
  574. /**
  575. * Get the last modified time of this entry.
  576. *
  577. * @return last modified time of this file
  578. * @since 5.1.9
  579. */
  580. public Instant getEntryLastModifiedInstant() {
  581. return current().getLastModifiedInstant();
  582. }
  583. /**
  584. * Obtain an input stream to read the file content.
  585. * <p>
  586. * Efficient implementations are not required. The caller will usually
  587. * obtain the stream only once per entry, if at all.
  588. * <p>
  589. * The input stream should not use buffering if the implementation can avoid
  590. * it. The caller will buffer as necessary to perform efficient block IO
  591. * operations.
  592. * <p>
  593. * The caller will close the stream once complete.
  594. *
  595. * @return a stream to read from the file.
  596. * @throws java.io.IOException
  597. * the file could not be opened for reading.
  598. */
  599. public InputStream openEntryStream() throws IOException {
  600. InputStream rawis = current().openInputStream();
  601. if (getCleanFilterCommand() == null
  602. && getEolStreamType() == EolStreamType.DIRECT) {
  603. return rawis;
  604. }
  605. return filterClean(rawis);
  606. }
  607. /**
  608. * Determine if the current entry path is ignored by an ignore rule.
  609. *
  610. * @return true if the entry was ignored by an ignore rule file.
  611. * @throws java.io.IOException
  612. * a relevant ignore rule file exists but cannot be read.
  613. */
  614. public boolean isEntryIgnored() throws IOException {
  615. return isEntryIgnored(pathLen);
  616. }
  617. /**
  618. * Determine if the entry path is ignored by an ignore rule.
  619. *
  620. * @param pLen
  621. * the length of the path in the path buffer.
  622. * @return true if the entry is ignored by an ignore rule.
  623. * @throws java.io.IOException
  624. * a relevant ignore rule file exists but cannot be read.
  625. */
  626. protected boolean isEntryIgnored(int pLen) throws IOException {
  627. return isEntryIgnored(pLen, mode);
  628. }
  629. /**
  630. * Determine if the entry path is ignored by an ignore rule.
  631. *
  632. * @param pLen
  633. * the length of the path in the path buffer.
  634. * @param fileMode
  635. * the original iterator file mode
  636. * @return true if the entry is ignored by an ignore rule.
  637. * @throws IOException
  638. * a relevant ignore rule file exists but cannot be read.
  639. */
  640. private boolean isEntryIgnored(int pLen, int fileMode)
  641. throws IOException {
  642. // The ignore code wants path to start with a '/' if possible.
  643. // If we have the '/' in our path buffer because we are inside
  644. // a sub-directory include it in the range we convert to string.
  645. //
  646. final int pOff = 0 < pathOffset ? pathOffset - 1 : pathOffset;
  647. String pathRel = TreeWalk.pathOf(this.path, pOff, pLen);
  648. String parentRel = getParentPath(pathRel);
  649. // CGit is processing .gitignore files by starting at the root of the
  650. // repository and then recursing into subdirectories. With this
  651. // approach, top-level ignored directories will be processed first which
  652. // allows to skip entire subtrees and further .gitignore-file processing
  653. // within these subtrees.
  654. //
  655. // We will follow the same approach by marking directories as "ignored"
  656. // here. This allows to have a simplified FastIgnore.checkIgnore()
  657. // implementation (both in terms of code and computational complexity):
  658. //
  659. // Without the "ignored" flag, we would have to apply the ignore-check
  660. // to a path and all of its parents always(!), to determine whether a
  661. // path is ignored directly or by one of its parent directories; with
  662. // the "ignored" flag, we know at this point that the parent directory
  663. // is definitely not ignored, thus the path can only become ignored if
  664. // there is a rule matching the path itself.
  665. if (isDirectoryIgnored(parentRel)) {
  666. return true;
  667. }
  668. IgnoreNode rules = getIgnoreNode();
  669. final Boolean ignored = rules != null
  670. ? rules.checkIgnored(pathRel, FileMode.TREE.equals(fileMode))
  671. : null;
  672. if (ignored != null) {
  673. return ignored.booleanValue();
  674. }
  675. return parent instanceof WorkingTreeIterator
  676. && ((WorkingTreeIterator) parent).isEntryIgnored(pLen,
  677. fileMode);
  678. }
  679. private IgnoreNode getIgnoreNode() throws IOException {
  680. if (ignoreNode instanceof PerDirectoryIgnoreNode)
  681. ignoreNode = ((PerDirectoryIgnoreNode) ignoreNode).load();
  682. return ignoreNode;
  683. }
  684. /**
  685. * Retrieves the {@link org.eclipse.jgit.attributes.AttributesNode} for the
  686. * current entry.
  687. *
  688. * @return the {@link org.eclipse.jgit.attributes.AttributesNode} for the
  689. * current entry.
  690. * @throws IOException
  691. */
  692. public AttributesNode getEntryAttributesNode() throws IOException {
  693. if (attributesNode instanceof PerDirectoryAttributesNode)
  694. attributesNode = ((PerDirectoryAttributesNode) attributesNode)
  695. .load();
  696. return attributesNode;
  697. }
  698. private static final Comparator<Entry> ENTRY_CMP = (Entry a,
  699. Entry b) -> Paths.compare(a.encodedName, 0, a.encodedNameLen,
  700. a.getMode().getBits(), b.encodedName, 0, b.encodedNameLen,
  701. b.getMode().getBits());
  702. /**
  703. * Constructor helper.
  704. *
  705. * @param list
  706. * files in the subtree of the work tree this iterator operates
  707. * on
  708. */
  709. protected void init(Entry[] list) {
  710. // Filter out nulls, . and .. as these are not valid tree entries,
  711. // also cache the encoded forms of the path names for efficient use
  712. // later on during sorting and iteration.
  713. //
  714. entries = list;
  715. int i, o;
  716. final CharsetEncoder nameEncoder = state.nameEncoder;
  717. for (i = 0, o = 0; i < entries.length; i++) {
  718. final Entry e = entries[i];
  719. if (e == null)
  720. continue;
  721. final String name = e.getName();
  722. if (".".equals(name) || "..".equals(name)) //$NON-NLS-1$ //$NON-NLS-2$
  723. continue;
  724. if (Constants.DOT_GIT.equals(name))
  725. continue;
  726. if (Constants.DOT_GIT_IGNORE.equals(name))
  727. ignoreNode = new PerDirectoryIgnoreNode(
  728. TreeWalk.pathOf(path, 0, pathOffset)
  729. + Constants.DOT_GIT_IGNORE,
  730. e);
  731. if (Constants.DOT_GIT_ATTRIBUTES.equals(name))
  732. attributesNode = new PerDirectoryAttributesNode(e);
  733. if (i != o)
  734. entries[o] = e;
  735. e.encodeName(nameEncoder);
  736. o++;
  737. }
  738. entryCnt = o;
  739. Arrays.sort(entries, 0, entryCnt, ENTRY_CMP);
  740. contentIdFromPtr = -1;
  741. ptr = 0;
  742. if (!eof())
  743. parseEntry();
  744. else if (pathLen == 0) // see bug 445363
  745. pathLen = pathOffset;
  746. }
  747. /**
  748. * Obtain the current entry from this iterator.
  749. *
  750. * @return the currently selected entry.
  751. */
  752. protected Entry current() {
  753. return entries[ptr];
  754. }
  755. /**
  756. * The result of a metadata-comparison between the current entry and a
  757. * {@link DirCacheEntry}
  758. */
  759. public enum MetadataDiff {
  760. /**
  761. * The entries are equal by metaData (mode, length,
  762. * modification-timestamp) or the <code>assumeValid</code> attribute of
  763. * the index entry is set
  764. */
  765. EQUAL,
  766. /**
  767. * The entries are not equal by metaData (mode, length) or the
  768. * <code>isUpdateNeeded</code> attribute of the index entry is set
  769. */
  770. DIFFER_BY_METADATA,
  771. /** index entry is smudged - can't use that entry for comparison */
  772. SMUDGED,
  773. /**
  774. * The entries are equal by metaData (mode, length) but differ by
  775. * modification-timestamp.
  776. */
  777. DIFFER_BY_TIMESTAMP
  778. }
  779. /**
  780. * Is the file mode of the current entry different than the given raw mode?
  781. *
  782. * @param rawMode
  783. * an int.
  784. * @return true if different, false otherwise
  785. */
  786. public boolean isModeDifferent(int rawMode) {
  787. // Determine difference in mode-bits of file and index-entry. In the
  788. // bitwise presentation of modeDiff we'll have a '1' when the two modes
  789. // differ at this position.
  790. int modeDiff = getEntryRawMode() ^ rawMode;
  791. if (modeDiff == 0)
  792. return false;
  793. // Do not rely on filemode differences in case of symbolic links
  794. if (getOptions().getSymLinks() == SymLinks.FALSE)
  795. if (FileMode.SYMLINK.equals(rawMode))
  796. return false;
  797. // Ignore the executable file bits if WorkingTreeOptions tell me to
  798. // do so. Ignoring is done by setting the bits representing a
  799. // EXECUTABLE_FILE to '0' in modeDiff
  800. if (!state.options.isFileMode())
  801. modeDiff &= ~FileMode.EXECUTABLE_FILE.getBits();
  802. return modeDiff != 0;
  803. }
  804. /**
  805. * Compare the metadata (mode, length, modification-timestamp) of the
  806. * current entry and a {@link org.eclipse.jgit.dircache.DirCacheEntry}
  807. *
  808. * @param entry
  809. * the {@link org.eclipse.jgit.dircache.DirCacheEntry} to compare
  810. * with
  811. * @return a
  812. * {@link org.eclipse.jgit.treewalk.WorkingTreeIterator.MetadataDiff}
  813. * which tells whether and how the entries metadata differ
  814. */
  815. public MetadataDiff compareMetadata(DirCacheEntry entry) {
  816. if (entry.isAssumeValid())
  817. return MetadataDiff.EQUAL;
  818. if (entry.isUpdateNeeded())
  819. return MetadataDiff.DIFFER_BY_METADATA;
  820. if (isModeDifferent(entry.getRawMode()))
  821. return MetadataDiff.DIFFER_BY_METADATA;
  822. // Don't check for length or lastmodified on folders
  823. int type = mode & FileMode.TYPE_MASK;
  824. if (type == FileMode.TYPE_TREE || type == FileMode.TYPE_GITLINK)
  825. return MetadataDiff.EQUAL;
  826. if (!entry.isSmudged() && entry.getLength() != (int) getEntryLength())
  827. return MetadataDiff.DIFFER_BY_METADATA;
  828. // Cache and file timestamps may differ in resolution. Therefore don't
  829. // compare instants directly but use a comparator that compares only
  830. // up to the lower apparent resolution of either timestamp.
  831. //
  832. // If core.checkstat is set to "minimal", compare only the seconds part.
  833. Instant cacheLastModified = entry.getLastModifiedInstant();
  834. Instant fileLastModified = getEntryLastModifiedInstant();
  835. if (timestampComparator.compare(cacheLastModified, fileLastModified,
  836. getOptions().getCheckStat() == CheckStat.MINIMAL) != 0) {
  837. return MetadataDiff.DIFFER_BY_TIMESTAMP;
  838. }
  839. if (entry.isSmudged()) {
  840. return MetadataDiff.SMUDGED;
  841. }
  842. // The file is clean when when comparing timestamps
  843. return MetadataDiff.EQUAL;
  844. }
  845. /**
  846. * Checks whether this entry differs from a given entry from the
  847. * {@link org.eclipse.jgit.dircache.DirCache}.
  848. *
  849. * File status information is used and if status is same we consider the
  850. * file identical to the state in the working directory. Native git uses
  851. * more stat fields than we have accessible in Java.
  852. *
  853. * @param entry
  854. * the entry from the dircache we want to compare against
  855. * @param forceContentCheck
  856. * True if the actual file content should be checked if
  857. * modification time differs.
  858. * @param reader
  859. * access to repository objects if necessary. Should not be null.
  860. * @return true if content is most likely different.
  861. * @throws java.io.IOException
  862. * @since 3.3
  863. */
  864. public boolean isModified(DirCacheEntry entry, boolean forceContentCheck,
  865. ObjectReader reader) throws IOException {
  866. if (entry == null)
  867. return !FileMode.MISSING.equals(getEntryFileMode());
  868. MetadataDiff diff = compareMetadata(entry);
  869. switch (diff) {
  870. case DIFFER_BY_TIMESTAMP:
  871. if (forceContentCheck) {
  872. // But we are told to look at content even though timestamps
  873. // tell us about modification
  874. return contentCheck(entry, reader);
  875. }
  876. // We are told to assume a modification if timestamps differs
  877. return true;
  878. case SMUDGED:
  879. // The file is clean by timestamps but the entry was smudged.
  880. // Lets do a content check
  881. return contentCheck(entry, reader);
  882. case EQUAL:
  883. if (mode == FileMode.SYMLINK.getBits()) {
  884. return contentCheck(entry, reader);
  885. }
  886. return false;
  887. case DIFFER_BY_METADATA:
  888. if (mode == FileMode.TREE.getBits()
  889. && entry.getFileMode().equals(FileMode.GITLINK)) {
  890. byte[] idBuffer = idBuffer();
  891. int idOffset = idOffset();
  892. if (entry.getObjectId().compareTo(idBuffer, idOffset) == 0) {
  893. return true;
  894. } else if (ObjectId.zeroId().compareTo(idBuffer,
  895. idOffset) == 0) {
  896. Path p = repository.getWorkTree().toPath()
  897. .resolve(entry.getPathString());
  898. return FileUtils.hasFiles(p);
  899. }
  900. return false;
  901. } else if (mode == FileMode.SYMLINK.getBits())
  902. return contentCheck(entry, reader);
  903. return true;
  904. default:
  905. throw new IllegalStateException(MessageFormat.format(
  906. JGitText.get().unexpectedCompareResult, diff.name()));
  907. }
  908. }
  909. /**
  910. * Get the file mode to use for the current entry when it is to be updated
  911. * in the index.
  912. *
  913. * @param indexIter
  914. * {@link org.eclipse.jgit.dircache.DirCacheIterator} positioned
  915. * at the same entry as this iterator or null if no
  916. * {@link org.eclipse.jgit.dircache.DirCacheIterator} is
  917. * available at this iterator's current entry
  918. * @return index file mode
  919. */
  920. public FileMode getIndexFileMode(DirCacheIterator indexIter) {
  921. final FileMode wtMode = getEntryFileMode();
  922. if (indexIter == null) {
  923. return wtMode;
  924. }
  925. final FileMode iMode = indexIter.getEntryFileMode();
  926. if (getOptions().isFileMode() && iMode != FileMode.GITLINK && iMode != FileMode.TREE) {
  927. return wtMode;
  928. }
  929. if (!getOptions().isFileMode()) {
  930. if (FileMode.REGULAR_FILE == wtMode
  931. && FileMode.EXECUTABLE_FILE == iMode) {
  932. return iMode;
  933. }
  934. if (FileMode.EXECUTABLE_FILE == wtMode
  935. && FileMode.REGULAR_FILE == iMode) {
  936. return iMode;
  937. }
  938. }
  939. if (FileMode.GITLINK == iMode
  940. && FileMode.TREE == wtMode && !getOptions().isDirNoGitLinks()) {
  941. return iMode;
  942. }
  943. if (FileMode.TREE == iMode
  944. && FileMode.GITLINK == wtMode) {
  945. return iMode;
  946. }
  947. return wtMode;
  948. }
  949. /**
  950. * Compares the entries content with the content in the filesystem.
  951. * Unsmudges the entry when it is detected that it is clean.
  952. *
  953. * @param entry
  954. * the entry to be checked
  955. * @param reader
  956. * acccess to repository data if necessary
  957. * @return <code>true</code> if the content doesn't match,
  958. * <code>false</code> if it matches
  959. * @throws IOException
  960. */
  961. private boolean contentCheck(DirCacheEntry entry, ObjectReader reader)
  962. throws IOException {
  963. if (getEntryObjectId().equals(entry.getObjectId())) {
  964. // Content has not changed
  965. // We know the entry can't be racily clean because it's still clean.
  966. // Therefore we unsmudge the entry!
  967. // If by any chance we now unsmudge although we are still in the
  968. // same time-slot as the last modification to the index file the
  969. // next index write operation will smudge again.
  970. // Caution: we are unsmudging just by setting the length of the
  971. // in-memory entry object. It's the callers task to detect that we
  972. // have modified the entry and to persist the modified index.
  973. entry.setLength((int) getEntryLength());
  974. return false;
  975. }
  976. if (mode == FileMode.SYMLINK.getBits()) {
  977. return !new File(readSymlinkTarget(current())).equals(
  978. new File(readContentAsNormalizedString(entry, reader)));
  979. }
  980. // Content differs: that's a real change
  981. return true;
  982. }
  983. private static String readContentAsNormalizedString(DirCacheEntry entry,
  984. ObjectReader reader) throws MissingObjectException, IOException {
  985. ObjectLoader open = reader.open(entry.getObjectId());
  986. byte[] cachedBytes = open.getCachedBytes();
  987. return FS.detect().normalize(RawParseUtils.decode(cachedBytes));
  988. }
  989. /**
  990. * Reads the target of a symlink as a string. This default implementation
  991. * fully reads the entry's input stream and converts it to a normalized
  992. * string. Subclasses may override to provide more specialized
  993. * implementations.
  994. *
  995. * @param entry
  996. * to read
  997. * @return the entry's content as a normalized string
  998. * @throws java.io.IOException
  999. * if the entry cannot be read or does not denote a symlink
  1000. * @since 4.6
  1001. */
  1002. protected String readSymlinkTarget(Entry entry) throws IOException {
  1003. if (!entry.getMode().equals(FileMode.SYMLINK)) {
  1004. throw new java.nio.file.NotLinkException(entry.getName());
  1005. }
  1006. long length = entry.getLength();
  1007. byte[] content = new byte[(int) length];
  1008. try (InputStream is = entry.openInputStream()) {
  1009. int bytesRead = IO.readFully(is, content, 0);
  1010. return FS.detect()
  1011. .normalize(RawParseUtils.decode(content, 0, bytesRead));
  1012. }
  1013. }
  1014. private static long computeLength(InputStream in) throws IOException {
  1015. // Since we only care about the length, use skip. The stream
  1016. // may be able to more efficiently wade through its data.
  1017. //
  1018. long length = 0;
  1019. for (;;) {
  1020. long n = in.skip(1 << 20);
  1021. if (n <= 0)
  1022. break;
  1023. length += n;
  1024. }
  1025. return length;
  1026. }
  1027. private byte[] computeHash(InputStream in, long length) throws IOException {
  1028. SHA1 contentDigest = SHA1.newInstance();
  1029. final byte[] contentReadBuffer = state.contentReadBuffer;
  1030. contentDigest.update(hblob);
  1031. contentDigest.update((byte) ' ');
  1032. long sz = length;
  1033. if (sz == 0) {
  1034. contentDigest.update((byte) '0');
  1035. } else {
  1036. final int bufn = contentReadBuffer.length;
  1037. int p = bufn;
  1038. do {
  1039. contentReadBuffer[--p] = digits[(int) (sz % 10)];
  1040. sz /= 10;
  1041. } while (sz > 0);
  1042. contentDigest.update(contentReadBuffer, p, bufn - p);
  1043. }
  1044. contentDigest.update((byte) 0);
  1045. for (;;) {
  1046. final int r = in.read(contentReadBuffer);
  1047. if (r <= 0)
  1048. break;
  1049. contentDigest.update(contentReadBuffer, 0, r);
  1050. sz += r;
  1051. }
  1052. if (sz != length)
  1053. return zeroid;
  1054. return contentDigest.digest();
  1055. }
  1056. /**
  1057. * A single entry within a working directory tree.
  1058. *
  1059. * @since 5.0
  1060. */
  1061. public abstract static class Entry {
  1062. byte[] encodedName;
  1063. int encodedNameLen;
  1064. void encodeName(CharsetEncoder enc) {
  1065. final ByteBuffer b;
  1066. try {
  1067. b = enc.encode(CharBuffer.wrap(getName()));
  1068. } catch (CharacterCodingException e) {
  1069. // This should so never happen.
  1070. throw new RuntimeException(MessageFormat.format(
  1071. JGitText.get().unencodeableFile, getName()), e);
  1072. }
  1073. encodedNameLen = b.limit();
  1074. if (b.hasArray() && b.arrayOffset() == 0)
  1075. encodedName = b.array();
  1076. else
  1077. b.get(encodedName = new byte[encodedNameLen]);
  1078. }
  1079. @Override
  1080. public String toString() {
  1081. return getMode().toString() + " " + getName(); //$NON-NLS-1$
  1082. }
  1083. /**
  1084. * Get the type of this entry.
  1085. * <p>
  1086. * <b>Note: Efficient implementation required.</b>
  1087. * <p>
  1088. * The implementation of this method must be efficient. If a subclass
  1089. * needs to compute the value they should cache the reference within an
  1090. * instance member instead.
  1091. *
  1092. * @return a file mode constant from {@link FileMode}.
  1093. */
  1094. public abstract FileMode getMode();
  1095. /**
  1096. * Get the byte length of this entry.
  1097. * <p>
  1098. * <b>Note: Efficient implementation required.</b>
  1099. * <p>
  1100. * The implementation of this method must be efficient. If a subclass
  1101. * needs to compute the value they should cache the reference within an
  1102. * instance member instead.
  1103. *
  1104. * @return size of this file, in bytes.
  1105. */
  1106. public abstract long getLength();
  1107. /**
  1108. * Get the last modified time of this entry.
  1109. * <p>
  1110. * <b>Note: Efficient implementation required.</b>
  1111. * <p>
  1112. * The implementation of this method must be efficient. If a subclass
  1113. * needs to compute the value they should cache the reference within an
  1114. * instance member instead.
  1115. *
  1116. * @return time since the epoch (in ms) of the last change.
  1117. * @deprecated use {@link #getLastModifiedInstant()} instead
  1118. */
  1119. @Deprecated
  1120. public abstract long getLastModified();
  1121. /**
  1122. * Get the last modified time of this entry.
  1123. * <p>
  1124. * <b>Note: Efficient implementation required.</b>
  1125. * <p>
  1126. * The implementation of this method must be efficient. If a subclass
  1127. * needs to compute the value they should cache the reference within an
  1128. * instance member instead.
  1129. *
  1130. * @return time of the last change.
  1131. * @since 5.1.9
  1132. */
  1133. public abstract Instant getLastModifiedInstant();
  1134. /**
  1135. * Get the name of this entry within its directory.
  1136. * <p>
  1137. * Efficient implementations are not required. The caller will obtain
  1138. * the name only once and cache it once obtained.
  1139. *
  1140. * @return name of the entry.
  1141. */
  1142. public abstract String getName();
  1143. /**
  1144. * Obtain an input stream to read the file content.
  1145. * <p>
  1146. * Efficient implementations are not required. The caller will usually
  1147. * obtain the stream only once per entry, if at all.
  1148. * <p>
  1149. * The input stream should not use buffering if the implementation can
  1150. * avoid it. The caller will buffer as necessary to perform efficient
  1151. * block IO operations.
  1152. * <p>
  1153. * The caller will close the stream once complete.
  1154. *
  1155. * @return a stream to read from the file.
  1156. * @throws IOException
  1157. * the file could not be opened for reading.
  1158. */
  1159. public abstract InputStream openInputStream() throws IOException;
  1160. }
  1161. /** Magic type indicating we know rules exist, but they aren't loaded. */
  1162. private static class PerDirectoryIgnoreNode extends IgnoreNode {
  1163. protected final Entry entry;
  1164. private final String name;
  1165. PerDirectoryIgnoreNode(String name, Entry entry) {
  1166. super(Collections.<FastIgnoreRule> emptyList());
  1167. this.name = name;
  1168. this.entry = entry;
  1169. }
  1170. IgnoreNode load() throws IOException {
  1171. IgnoreNode r = new IgnoreNode();
  1172. try (InputStream in = entry.openInputStream()) {
  1173. r.parse(name, in);
  1174. }
  1175. return r.getRules().isEmpty() ? null : r;
  1176. }
  1177. }
  1178. /** Magic type indicating there may be rules for the top level. */
  1179. private static class RootIgnoreNode extends PerDirectoryIgnoreNode {
  1180. final Repository repository;
  1181. RootIgnoreNode(Entry entry, Repository repository) {
  1182. super(entry != null ? entry.getName() : null, entry);
  1183. this.repository = repository;
  1184. }
  1185. @Override
  1186. IgnoreNode load() throws IOException {
  1187. IgnoreNode r;
  1188. if (entry != null) {
  1189. r = super.load();
  1190. if (r == null)
  1191. r = new IgnoreNode();
  1192. } else {
  1193. r = new IgnoreNode();
  1194. }
  1195. FS fs = repository.getFS();
  1196. Path path = repository.getConfig().getPath(
  1197. ConfigConstants.CONFIG_CORE_SECTION, null,
  1198. ConfigConstants.CONFIG_KEY_EXCLUDESFILE, fs, null, null);
  1199. if (path != null) {
  1200. loadRulesFromFile(r, path.toFile());
  1201. }
  1202. File exclude = fs.resolve(repository.getDirectory(),
  1203. Constants.INFO_EXCLUDE);
  1204. loadRulesFromFile(r, exclude);
  1205. return r.getRules().isEmpty() ? null : r;
  1206. }
  1207. private static void loadRulesFromFile(IgnoreNode r, File exclude)
  1208. throws FileNotFoundException, IOException {
  1209. if (FS.DETECTED.exists(exclude)) {
  1210. try (FileInputStream in = new FileInputStream(exclude)) {
  1211. r.parse(exclude.getAbsolutePath(), in);
  1212. }
  1213. }
  1214. }
  1215. }
  1216. /** Magic type indicating we know rules exist, but they aren't loaded. */
  1217. private static class PerDirectoryAttributesNode extends AttributesNode {
  1218. final Entry entry;
  1219. PerDirectoryAttributesNode(Entry entry) {
  1220. super(Collections.<AttributesRule> emptyList());
  1221. this.entry = entry;
  1222. }
  1223. AttributesNode load() throws IOException {
  1224. AttributesNode r = new AttributesNode();
  1225. try (InputStream in = entry.openInputStream()) {
  1226. r.parse(in);
  1227. }
  1228. return r.getRules().isEmpty() ? null : r;
  1229. }
  1230. }
  1231. private static final class IteratorState {
  1232. /** Options used to process the working tree. */
  1233. final WorkingTreeOptions options;
  1234. /** File name character encoder. */
  1235. final CharsetEncoder nameEncoder;
  1236. /** Buffer used to perform {@link #contentId} computations. */
  1237. byte[] contentReadBuffer;
  1238. /** TreeWalk with a (supposedly) matching DirCacheIterator. */
  1239. TreeWalk walk;
  1240. /** Position of the matching {@link DirCacheIterator}. */
  1241. int dirCacheTree = -1;
  1242. /** Whether the iterator shall walk ignored directories. */
  1243. boolean walkIgnored = false;
  1244. final Map<String, Boolean> directoryToIgnored = new HashMap<>();
  1245. IteratorState(WorkingTreeOptions options) {
  1246. this.options = options;
  1247. this.nameEncoder = UTF_8.newEncoder();
  1248. }
  1249. void initializeReadBuffer() {
  1250. if (contentReadBuffer == null) {
  1251. contentReadBuffer = new byte[BUFFER_SIZE];
  1252. }
  1253. }
  1254. }
  1255. /**
  1256. * Get the clean filter command for the current entry.
  1257. *
  1258. * @return the clean filter command for the current entry or
  1259. * <code>null</code> if no such command is defined
  1260. * @throws java.io.IOException
  1261. * @since 4.2
  1262. */
  1263. public String getCleanFilterCommand() throws IOException {
  1264. if (cleanFilterCommandHolder == null) {
  1265. String cmd = null;
  1266. if (state.walk != null) {
  1267. cmd = state.walk
  1268. .getFilterCommand(Constants.ATTR_FILTER_TYPE_CLEAN);
  1269. }
  1270. cleanFilterCommandHolder = new Holder<>(cmd);
  1271. }
  1272. return cleanFilterCommandHolder.get();
  1273. }
  1274. /**
  1275. * Get the eol stream type for the current entry.
  1276. *
  1277. * @return the eol stream type for the current entry or <code>null</code> if
  1278. * it cannot be determined. When state or state.walk is null or the
  1279. * {@link org.eclipse.jgit.treewalk.TreeWalk} is not based on a
  1280. * {@link org.eclipse.jgit.lib.Repository} then null is returned.
  1281. * @throws java.io.IOException
  1282. * @since 4.3
  1283. */
  1284. public EolStreamType getEolStreamType() throws IOException {
  1285. return getEolStreamType(null);
  1286. }
  1287. /**
  1288. * @param opType
  1289. * The operationtype (checkin/checkout) which should be used
  1290. * @return the eol stream type for the current entry or <code>null</code> if
  1291. * it cannot be determined. When state or state.walk is null or the
  1292. * {@link TreeWalk} is not based on a {@link Repository} then null
  1293. * is returned.
  1294. * @throws IOException
  1295. */
  1296. private EolStreamType getEolStreamType(OperationType opType)
  1297. throws IOException {
  1298. if (eolStreamTypeHolder == null) {
  1299. EolStreamType type = null;
  1300. if (state.walk != null) {
  1301. type = state.walk.getEolStreamType(opType);
  1302. OperationType operationType = opType != null ? opType
  1303. : state.walk.getOperationType();
  1304. if (OperationType.CHECKIN_OP.equals(operationType)
  1305. && EolStreamType.AUTO_LF.equals(type)
  1306. && hasCrLfInIndex(getDirCacheIterator())) {
  1307. // If text=auto (or core.autocrlf=true) and the file has
  1308. // already been committed with CR/LF, then don't convert.
  1309. type = EolStreamType.DIRECT;
  1310. }
  1311. } else {
  1312. switch (getOptions().getAutoCRLF()) {
  1313. case FALSE:
  1314. type = EolStreamType.DIRECT;
  1315. break;
  1316. case TRUE:
  1317. case INPUT:
  1318. type = EolStreamType.AUTO_LF;
  1319. break;
  1320. }
  1321. }
  1322. eolStreamTypeHolder = new Holder<>(type);
  1323. }
  1324. return eolStreamTypeHolder.get();
  1325. }
  1326. /**
  1327. * Determines whether the file was committed un-normalized. If the iterator
  1328. * points to a conflict entry, checks the "ours" version.
  1329. *
  1330. * @param dirCache
  1331. * iterator pointing to the current entry for the file in the
  1332. * index
  1333. * @return {@code true} if the file in the index is not binary and has CR/LF
  1334. * line endings, {@code false} otherwise
  1335. */
  1336. private boolean hasCrLfInIndex(DirCacheIterator dirCache) {
  1337. if (dirCache == null) {
  1338. return false;
  1339. }
  1340. // Read blob from index and check for CR/LF-delimited text.
  1341. DirCacheEntry entry = dirCache.getDirCacheEntry();
  1342. if ((entry.getRawMode() & FileMode.TYPE_MASK) == FileMode.TYPE_FILE) {
  1343. ObjectId blobId = entry.getObjectId();
  1344. if (entry.getStage() > 0
  1345. && entry.getStage() != DirCacheEntry.STAGE_2) {
  1346. blobId = null;
  1347. // Merge conflict: check ours (stage 2)
  1348. byte[] name = entry.getRawPath();
  1349. int i = 0;
  1350. while (!dirCache.eof()) {
  1351. dirCache.next(1);
  1352. i++;
  1353. entry = dirCache.getDirCacheEntry();
  1354. if (entry == null
  1355. || !Arrays.equals(name, entry.getRawPath())) {
  1356. break;
  1357. }
  1358. if (entry.getStage() == DirCacheEntry.STAGE_2) {
  1359. if ((entry.getRawMode()
  1360. & FileMode.TYPE_MASK) == FileMode.TYPE_FILE) {
  1361. blobId = entry.getObjectId();
  1362. }
  1363. break;
  1364. }
  1365. }
  1366. dirCache.back(i);
  1367. }
  1368. if (blobId != null) {
  1369. try (ObjectReader reader = repository.newObjectReader()) {
  1370. ObjectLoader loader = reader.open(blobId,
  1371. Constants.OBJ_BLOB);
  1372. try {
  1373. return RawText.isCrLfText(loader.getCachedBytes());
  1374. } catch (LargeObjectException e) {
  1375. try (InputStream in = loader.openStream()) {
  1376. return RawText.isCrLfText(in);
  1377. }
  1378. }
  1379. } catch (IOException e) {
  1380. // Ignore and return false below
  1381. }
  1382. }
  1383. }
  1384. return false;
  1385. }
  1386. private boolean isDirectoryIgnored(String pathRel) throws IOException {
  1387. final int pOff = 0 < pathOffset ? pathOffset - 1 : pathOffset;
  1388. final String base = TreeWalk.pathOf(this.path, 0, pOff);
  1389. final String pathAbs = concatPath(base, pathRel);
  1390. return isDirectoryIgnored(pathRel, pathAbs);
  1391. }
  1392. private boolean isDirectoryIgnored(String pathRel, String pathAbs)
  1393. throws IOException {
  1394. assert pathRel.length() == 0 || (pathRel.charAt(0) != '/'
  1395. && pathRel.charAt(pathRel.length() - 1) != '/');
  1396. assert pathAbs.length() == 0 || (pathAbs.charAt(0) != '/'
  1397. && pathAbs.charAt(pathAbs.length() - 1) != '/');
  1398. assert pathAbs.endsWith(pathRel);
  1399. Boolean ignored = state.directoryToIgnored.get(pathAbs);
  1400. if (ignored != null) {
  1401. return ignored.booleanValue();
  1402. }
  1403. final String parentRel = getParentPath(pathRel);
  1404. if (parentRel != null && isDirectoryIgnored(parentRel)) {
  1405. state.directoryToIgnored.put(pathAbs, Boolean.TRUE);
  1406. return true;
  1407. }
  1408. final IgnoreNode node = getIgnoreNode();
  1409. for (String p = pathRel; node != null
  1410. && !"".equals(p); p = getParentPath(p)) { //$NON-NLS-1$
  1411. ignored = node.checkIgnored(p, true);
  1412. if (ignored != null) {
  1413. state.directoryToIgnored.put(pathAbs, ignored);
  1414. return ignored.booleanValue();
  1415. }
  1416. }
  1417. if (!(this.parent instanceof WorkingTreeIterator)) {
  1418. state.directoryToIgnored.put(pathAbs, Boolean.FALSE);
  1419. return false;
  1420. }
  1421. final WorkingTreeIterator wtParent = (WorkingTreeIterator) this.parent;
  1422. final String parentRelPath = concatPath(
  1423. TreeWalk.pathOf(this.path, wtParent.pathOffset, pathOffset - 1),
  1424. pathRel);
  1425. assert concatPath(TreeWalk.pathOf(wtParent.path, 0,
  1426. Math.max(0, wtParent.pathOffset - 1)), parentRelPath)
  1427. .equals(pathAbs);
  1428. return wtParent.isDirectoryIgnored(parentRelPath, pathAbs);
  1429. }
  1430. private static String getParentPath(String path) {
  1431. final int slashIndex = path.lastIndexOf('/', path.length() - 2);
  1432. if (slashIndex > 0) {
  1433. return path.substring(path.charAt(0) == '/' ? 1 : 0, slashIndex);
  1434. }
  1435. return path.length() > 0 ? "" : null; //$NON-NLS-1$
  1436. }
  1437. private static String concatPath(String p1, String p2) {
  1438. return p1 + (p1.length() > 0 && p2.length() > 0 ? "/" : "") + p2; //$NON-NLS-1$ //$NON-NLS-2$
  1439. }
  1440. }