jgit/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java

/*
 * Copyright (C) 2008, 2010 Google Inc.
 * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com>
 * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
 * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others
 *
 * This program and the accompanying materials are made available under the
 * terms of the Eclipse Distribution License v. 1.0 which is available at
 * https://www.eclipse.org/org/documents/edl-v10.php.
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */

package org.eclipse.jgit.transport;

import static org.eclipse.jgit.transport.GitProtocolConstants.COMMAND_LS_REFS;
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT;
import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_PEELED;
import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_SYMREF_TARGET;
import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_1;
import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_2;

import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.MessageFormat;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

import org.eclipse.jgit.annotations.NonNull;
import org.eclipse.jgit.errors.InvalidObjectIdException;
import org.eclipse.jgit.errors.NoRemoteRepositoryException;
import org.eclipse.jgit.errors.PackProtocolException;
import org.eclipse.jgit.errors.RemoteRepositoryException;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.SymbolicRef;
import org.eclipse.jgit.util.StringUtils;
import org.eclipse.jgit.util.io.InterruptTimer;
import org.eclipse.jgit.util.io.TimeoutInputStream;
import org.eclipse.jgit.util.io.TimeoutOutputStream;

/**
 * Base helper class for pack-based operations implementations. Provides partial
 * implementation of pack-protocol - refs advertising and capabilities support,
 * and some other helper methods.
 *
 * @see BasePackFetchConnection
 * @see BasePackPushConnection
 */
abstract class BasePackConnection extends BaseConnection {

	protected static final String CAPABILITY_SYMREF_PREFIX = "symref="; //$NON-NLS-1$

	/** The repository this transport fetches into, or pushes out of. */
	protected final Repository local;

	/** Remote repository location. */
	protected final URIish uri;

	/** A transport connected to {@link #uri}. */
	protected final Transport transport;

	/** Low-level input stream, if a timeout was configured. */
	protected TimeoutInputStream timeoutIn;

	/** Low-level output stream, if a timeout was configured. */
	protected TimeoutOutputStream timeoutOut;

	/** Timer to manage {@link #timeoutIn} and {@link #timeoutOut}. */
	private InterruptTimer myTimer;

	/** Input stream reading from the remote. */
	protected InputStream in;

	/** Output stream sending to the remote. */
	protected OutputStream out;

	/** Packet line decoder around {@link #in}. */
	protected PacketLineIn pckIn;

	/** Packet line encoder around {@link #out}. */
	protected PacketLineOut pckOut;

	/** Send {@link PacketLineOut#end()} before closing {@link #out}? */
	protected boolean outNeedsEnd;

	/** True if this is a stateless RPC connection. */
	protected boolean statelessRPC;

	/** Capability tokens advertised by the remote side. */
	private final Map<String, String> remoteCapabilities = new HashMap<>();

	/** Extra objects the remote has, but which aren't offered as refs. */
	protected final Set<ObjectId> additionalHaves = new HashSet<>();

	private TransferConfig.ProtocolVersion protocol = TransferConfig.ProtocolVersion.V0;

	BasePackConnection(PackTransport packTransport) {
		transport = (Transport) packTransport;
		local = transport.local;
		uri = transport.uri;
	}

	TransferConfig.ProtocolVersion getProtocolVersion() {
		return protocol;
	}

	void setProtocolVersion(@NonNull TransferConfig.ProtocolVersion protocol) {
		this.protocol = protocol;
	}

	/**
	 * Configure this connection with the directional pipes.
	 *
	 * @param myIn
	 *            input stream to receive data from the peer. Caller must ensure
	 *            the input is buffered, otherwise read performance may suffer.
	 * @param myOut
	 *            output stream to transmit data to the peer. Caller must ensure
	 *            the output is buffered, otherwise write performance may
	 *            suffer.
	 */
	protected final void init(InputStream myIn, OutputStream myOut) {
		final int timeout = transport.getTimeout();
		if (timeout > 0) {
			final Thread caller = Thread.currentThread();
			if (myTimer == null) {
				myTimer = new InterruptTimer(caller.getName() + "-Timer"); //$NON-NLS-1$
			}
			timeoutIn = new TimeoutInputStream(myIn, myTimer);
			timeoutOut = new TimeoutOutputStream(myOut, myTimer);
			timeoutIn.setTimeout(timeout * 1000);
			timeoutOut.setTimeout(timeout * 1000);
			myIn = timeoutIn;
			myOut = timeoutOut;
		}

		in = myIn;
		out = myOut;

		pckIn = new PacketLineIn(in);
		pckOut = new PacketLineOut(out);
		outNeedsEnd = true;
	}

	/**
	 * Reads the advertised references through the initialized stream.
	 * <p>
	 * Subclass implementations may call this method only after setting up the
	 * input and output streams with {@link #init(InputStream, OutputStream)}.
	 * <p>
	 * If any errors occur, this connection is automatically closed by invoking
	 * {@link #close()} and the exception is wrapped (if necessary) and thrown
	 * as a {@link org.eclipse.jgit.errors.TransportException}.
	 *
	 * @return {@code true} if the refs were read; {@code false} otherwise
	 *         indicating that {@link #lsRefs} must be called
	 *
	 * @throws org.eclipse.jgit.errors.TransportException
	 *             the reference list could not be scanned.
	 */
	protected boolean readAdvertisedRefs() throws TransportException {
		try {
			return readAdvertisedRefsImpl();
		} catch (TransportException err) {
			close();
			throw err;
		} catch (IOException | RuntimeException err) {
			close();
			throw new TransportException(err.getMessage(), err);
		}
	}

	private String readLine() throws IOException {
		String line = pckIn.readString();
		if (PacketLineIn.isEnd(line)) {
			return null;
		}
		if (line.startsWith("ERR ")) { //$NON-NLS-1$
			// This is a customized remote service error.
			// Users should be informed about it.
			throw new RemoteRepositoryException(uri, line.substring(4));
		}
		return line;
	}

	private boolean readAdvertisedRefsImpl() throws IOException {
		final Map<String, Ref> avail = new LinkedHashMap<>();
		final Map<String, String> symRefs = new LinkedHashMap<>();
		for (boolean first = true;; first = false) {
			String line;

			if (first) {
				boolean isV1 = false;
				try {
					line = readLine();
				} catch (EOFException e) {
					TransportException noRepo = noRepository();
					noRepo.initCause(e);
					throw noRepo;
				}
				if (line != null && VERSION_1.equals(line)) {
					// Same as V0, except for this extra line. We shouldn't get
					// it since we never request V1.
					setProtocolVersion(TransferConfig.ProtocolVersion.V0);
					isV1 = true;
					line = readLine();
				}
				if (line == null) {
					break;
				}
				final int nul = line.indexOf('\0');
				if (nul >= 0) {
					// Protocol V0: The first line (if any) may contain
					// "hidden" capability values after a NUL byte.
					for (String capability : line.substring(nul + 1)
							.split(" ")) { //$NON-NLS-1$
						if (capability.startsWith(CAPABILITY_SYMREF_PREFIX)) {
							String[] parts = capability
									.substring(
											CAPABILITY_SYMREF_PREFIX.length())
									.split(":", 2); //$NON-NLS-1$
							if (parts.length == 2) {
								symRefs.put(parts[0], parts[1]);
							}
						} else {
							addCapability(capability);
						}
					}
					line = line.substring(0, nul);
					setProtocolVersion(TransferConfig.ProtocolVersion.V0);
				} else if (!isV1 && VERSION_2.equals(line)) {
					// Protocol V2: remaining lines are capabilities as
					// key=value pairs
					setProtocolVersion(TransferConfig.ProtocolVersion.V2);
					readCapabilitiesV2();
					// Break out here so that stateless RPC transports get a
					// chance to set up the output stream.
					return false;
				} else {
					setProtocolVersion(TransferConfig.ProtocolVersion.V0);
				}
			} else {
				line = readLine();
				if (line == null) {
					break;
				}
			}

			// Expecting to get a line in the form "sha1 refname"
			if (line.length() < 41 || line.charAt(40) != ' ') {
				throw invalidRefAdvertisementLine(line);
			}
			String name = line.substring(41, line.length());
			if (first && name.equals("capabilities^{}")) { //$NON-NLS-1$
				// special line from git-receive-pack (protocol V0) to show
				// capabilities when there are no refs to advertise
				continue;
			}

			final ObjectId id = toId(line, line.substring(0, 40));
			if (name.equals(".have")) { //$NON-NLS-1$
				additionalHaves.add(id);
			} else {
				processLineV1(name, id, avail);
			}
		}
		updateWithSymRefs(avail, symRefs);
		available(avail);
		return true;
	}

	/**
	 * Issue a protocol V2 ls-refs command and read its response.
	 *
	 * @param refSpecs
	 *            to produce ref prefixes from if the server supports git
	 *            protocol V2
	 * @param additionalPatterns
	 *            to use for ref prefixes if the server supports git protocol V2
	 * @throws TransportException
	 *             if the command could not be run or its output not be read
	 */
	protected void lsRefs(Collection<RefSpec> refSpecs,
			String... additionalPatterns) throws TransportException {
		try {
			lsRefsImpl(refSpecs, additionalPatterns);
		} catch (TransportException err) {
			close();
			throw err;
		} catch (IOException | RuntimeException err) {
			close();
			throw new TransportException(err.getMessage(), err);
		}
	}

	private void lsRefsImpl(Collection<RefSpec> refSpecs,
			String... additionalPatterns) throws IOException {
		pckOut.writeString("command=" + COMMAND_LS_REFS); //$NON-NLS-1$
		// Add the user-agent
		String agent = UserAgent.get();
		if (agent != null && isCapableOf(OPTION_AGENT)) {
			pckOut.writeString(OPTION_AGENT + '=' + agent);
		}
		pckOut.writeDelim();
		pckOut.writeString("peel"); //$NON-NLS-1$
		pckOut.writeString("symrefs"); //$NON-NLS-1$
		for (String refPrefix : getRefPrefixes(refSpecs, additionalPatterns)) {
			pckOut.writeString("ref-prefix " + refPrefix); //$NON-NLS-1$
		}
		pckOut.end();
		final Map<String, Ref> avail = new LinkedHashMap<>();
		final Map<String, String> symRefs = new LinkedHashMap<>();
		for (;;) {
			String line = readLine();
			if (line == null) {
				break;
			}
			// Expecting to get a line in the form "sha1 refname"
			if (line.length() < 41 || line.charAt(40) != ' ') {
				throw invalidRefAdvertisementLine(line);
			}
			String name = line.substring(41, line.length());
			final ObjectId id = toId(line, line.substring(0, 40));
			if (name.equals(".have")) { //$NON-NLS-1$
				additionalHaves.add(id);
			} else {
				processLineV2(line, id, name, avail, symRefs);
			}
		}
		updateWithSymRefs(avail, symRefs);
		available(avail);
	}

	private Collection<String> getRefPrefixes(Collection<RefSpec> refSpecs,
			String... additionalPatterns) {
		if (refSpecs.isEmpty() && (additionalPatterns == null
				|| additionalPatterns.length == 0)) {
			return Collections.emptyList();
		}
		Set<String> patterns = new HashSet<>();
		if (additionalPatterns != null) {
			Arrays.stream(additionalPatterns).filter(Objects::nonNull)
					.forEach(patterns::add);
		}
		for (RefSpec spec : refSpecs) {
			// TODO: for now we only do protocol V2 for fetch. For push
			// RefSpecs, the logic would need to be different. (At the
			// minimum, take spec.getDestination().)
			String src = spec.getSource();
			if (ObjectId.isId(src)) {
				continue;
			}
			if (spec.isWildcard()) {
				patterns.add(src.substring(0, src.indexOf('*')));
			} else {
				patterns.add(src);
				patterns.add(Constants.R_REFS + src);
				patterns.add(Constants.R_HEADS + src);
				patterns.add(Constants.R_TAGS + src);
			}
		}
		return patterns;
	}

	private void readCapabilitiesV2() throws IOException {
		// In git protocol V2, capabilities are different. If it's a key-value
		// pair, the key may be a command name, and the value a space-separated
		// list of capabilities for that command. We still store it in the same
		// map as for protocol v0/v1. Protocol v2 code has to account for this.
		for (;;) {
			String line = readLine();
			if (line == null) {
				break;
			}
			addCapability(line);
		}
	}

	private void addCapability(String capability) {
		String parts[] = capability.split("=", 2); //$NON-NLS-1$
		if (parts.length == 2) {
			remoteCapabilities.put(parts[0], parts[1]);
		}
		remoteCapabilities.put(capability, null);
	}

	private ObjectId toId(String line, String value)
			throws PackProtocolException {
		try {
			return ObjectId.fromString(value);
		} catch (InvalidObjectIdException e) {
			PackProtocolException ppe = invalidRefAdvertisementLine(line);
			ppe.initCause(e);
			throw ppe;
		}
	}

	private void processLineV1(String name, ObjectId id, Map<String, Ref> avail)
			throws IOException {
		if (name.endsWith("^{}")) { //$NON-NLS-1$
			name = name.substring(0, name.length() - 3);
			final Ref prior = avail.get(name);
			if (prior == null) {
				throw new PackProtocolException(uri, MessageFormat.format(
						JGitText.get().advertisementCameBefore, name, name));
			}
			if (prior.getPeeledObjectId() != null) {
				throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$
			}
			avail.put(name, new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name,
					prior.getObjectId(), id));
		} else {
			final Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag(
					Ref.Storage.NETWORK, name, id));
			if (prior != null) {
				throw duplicateAdvertisement(name);
			}
		}
	}

	private void processLineV2(String line, ObjectId id, String rest,
			Map<String, Ref> avail, Map<String, String> symRefs)
			throws IOException {
		String[] parts = rest.split(" "); //$NON-NLS-1$
		String name = parts[0];
		// Two attributes possible, symref-target or peeled
		String symRefTarget = null;
		String peeled = null;
		for (int i = 1; i < parts.length; i++) {
			if (parts[i].startsWith(REF_ATTR_SYMREF_TARGET)) {
				if (symRefTarget != null) {
					throw new PackProtocolException(uri, MessageFormat.format(
							JGitText.get().duplicateRefAttribute, line));
				}
				symRefTarget = parts[i]
						.substring(REF_ATTR_SYMREF_TARGET.length());
			} else if (parts[i].startsWith(REF_ATTR_PEELED)) {
				if (peeled != null) {
					throw new PackProtocolException(uri, MessageFormat.format(
							JGitText.get().duplicateRefAttribute, line));
				}
				peeled = parts[i].substring(REF_ATTR_PEELED.length());
			}
			if (peeled != null && symRefTarget != null) {
				break;
			}
		}
		Ref idRef;
		if (peeled != null) {
			idRef = new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name, id,
					toId(line, peeled));
		} else {
			idRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, name, id);
		}
		Ref prior = avail.put(name, idRef);
		if (prior != null) {
			throw duplicateAdvertisement(name);
		}
		if (!StringUtils.isEmptyOrNull(symRefTarget)) {
			symRefs.put(name, symRefTarget);
		}
	}

	/**
	 * Updates the given refMap with {@link SymbolicRef}s defined by the given
	 * symRefs.
	 * <p>
	 * For each entry, symRef, in symRefs, whose value is a key in refMap, adds
	 * a new entry to refMap with that same key and value of a new
	 * {@link SymbolicRef} with source=symRef.key and
	 * target=refMap.get(symRef.value), then removes that entry from symRefs.
	 * <p>
	 * If refMap already contains an entry for symRef.key, it is replaced.
	 * </p>
	 * </p>
	 * <p>
	 * For example, given:
	 * </p>
	 *
	 * <pre>
	 * refMap.put("refs/heads/main", ref);
	 * symRefs.put("HEAD", "refs/heads/main");
	 * </pre>
	 *
	 * then:
	 *
	 * <pre>
	 * updateWithSymRefs(refMap, symRefs);
	 * </pre>
	 *
	 * has the <em>effect</em> of:
	 *
	 * <pre>
	 * refMap.put("HEAD",
	 * 		new SymbolicRef("HEAD", refMap.get(symRefs.remove("HEAD"))))
	 * </pre>
	 * <p>
	 * Any entry in symRefs whose value is not a key in refMap is ignored. Any
	 * circular symRefs are ignored.
	 * </p>
	 * <p>
	 * Upon completion, symRefs will contain only any unresolvable entries.
	 * </p>
	 *
	 * @param refMap
	 *            a non-null, modifiable, Map to update, and the provider of
	 *            symref targets.
	 * @param symRefs
	 *            a non-null, modifiable, Map of symrefs.
	 * @throws NullPointerException
	 *             if refMap or symRefs is null
	 */
	static void updateWithSymRefs(Map<String, Ref> refMap, Map<String, String> symRefs) {
		boolean haveNewRefMapEntries = !refMap.isEmpty();
		while (!symRefs.isEmpty() && haveNewRefMapEntries) {
			haveNewRefMapEntries = false;
			final Iterator<Map.Entry<String, String>> iterator = symRefs.entrySet().iterator();
			while (iterator.hasNext()) {
				final Map.Entry<String, String> symRef = iterator.next();
				if (!symRefs.containsKey(symRef.getValue())) { // defer forward reference
					final Ref r = refMap.get(symRef.getValue());
					if (r != null) {
						refMap.put(symRef.getKey(), new SymbolicRef(symRef.getKey(), r));
						haveNewRefMapEntries = true;
						iterator.remove();
					}
				}
			}
		}
		// If HEAD is still in the symRefs map here, the real ref was not
		// reported, but we know it must point to the object reported for HEAD.
		// So fill it in in the refMap.
		String headRefName = symRefs.get(Constants.HEAD);
		if (headRefName != null && !refMap.containsKey(headRefName)) {
			Ref headRef = refMap.get(Constants.HEAD);
			if (headRef != null) {
				ObjectId headObj = headRef.getObjectId();
				headRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK,
						headRefName, headObj);
				refMap.put(headRefName, headRef);
				headRef = new SymbolicRef(Constants.HEAD, headRef);
				refMap.put(Constants.HEAD, headRef);
				symRefs.remove(Constants.HEAD);
			}
		}
	}

	/**
	 * Create an exception to indicate problems finding a remote repository. The
	 * caller is expected to throw the returned exception.
	 *
	 * Subclasses may override this method to provide better diagnostics.
	 *
	 * @return a TransportException saying a repository cannot be found and
	 *         possibly why.
	 */
	protected TransportException noRepository() {
		return new NoRemoteRepositoryException(uri, JGitText.get().notFound);
	}

	/**
	 * Whether this option is supported
	 *
	 * @param option
	 *            option string
	 * @return whether this option is supported
	 */
	protected boolean isCapableOf(String option) {
		return remoteCapabilities.containsKey(option);
	}

	/**
	 * Request capability
	 *
	 * @param b
	 *            buffer
	 * @param option
	 *            option we want
	 * @return {@code true} if the requested option is supported
	 */
	protected boolean wantCapability(StringBuilder b, String option) {
		if (!isCapableOf(option))
			return false;
		b.append(' ');
		b.append(option);
		return true;
	}

	/**
	 * Return a capability value.
	 *
	 * @param option
	 *            to get
	 * @return the value stored, if any.
	 */
	protected String getCapability(String option) {
		return remoteCapabilities.get(option);
	}

	/**
	 * Add user agent capability
	 *
	 * @param b
	 *            a {@link java.lang.StringBuilder} object.
	 */
	protected void addUserAgentCapability(StringBuilder b) {
		String a = UserAgent.get();
		if (a != null && remoteCapabilities.get(OPTION_AGENT) != null) {
			b.append(' ').append(OPTION_AGENT).append('=').append(a);
		}
	}

	/** {@inheritDoc} */
	@Override
	public String getPeerUserAgent() {
		String agent = remoteCapabilities.get(OPTION_AGENT);
		return agent != null ? agent : super.getPeerUserAgent();
	}

	private PackProtocolException duplicateAdvertisement(String name) {
		return new PackProtocolException(uri, MessageFormat.format(JGitText.get().duplicateAdvertisementsOf, name));
	}

	private PackProtocolException invalidRefAdvertisementLine(String line) {
		return new PackProtocolException(uri, MessageFormat.format(JGitText.get().invalidRefAdvertisementLine, line));
	}

	/** {@inheritDoc} */
	@Override
	public void close() {
		if (out != null) {
			try {
				if (outNeedsEnd) {
					outNeedsEnd = false;
					pckOut.end();
				}
				out.close();
			} catch (IOException err) {
				// Ignore any close errors.
			} finally {
				out = null;
				pckOut = null;
			}
		}

		if (in != null) {
			try {
				in.close();
			} catch (IOException err) {
				// Ignore any close errors.
			} finally {
				in = null;
				pckIn = null;
			}
		}

		if (myTimer != null) {
			try {
				myTimer.terminate();
			} finally {
				myTimer = null;
				timeoutIn = null;
				timeoutOut = null;
			}
		}
	}

	/**
	 * Tell the peer we are disconnecting, if it cares to know.
	 */
	protected void endOut() {
		if (outNeedsEnd && out != null) {
			try {
				outNeedsEnd = false;
				pckOut.end();
			} catch (IOException e) {
				try {
					out.close();
				} catch (IOException err) {
					// Ignore any close errors.
				} finally {
					out = null;
					pckOut = null;
				}
			}
		}
	}
}