diff options
Diffstat (limited to 'src/main/java/com/gitblit/instance')
4 files changed, 647 insertions, 0 deletions
diff --git a/src/main/java/com/gitblit/instance/GitblitInstance.java b/src/main/java/com/gitblit/instance/GitblitInstance.java new file mode 100644 index 00000000..ca1fda38 --- /dev/null +++ b/src/main/java/com/gitblit/instance/GitblitInstance.java @@ -0,0 +1,212 @@ +package com.gitblit.instance; + +import com.gitblit.IStoredSettings; +import com.gitblit.manager.IRuntimeManager; +import com.gitblit.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static com.gitblit.utils.JsonUtils.sendJsonString; + +public class GitblitInstance +{ + private final static String STATS_URL = "https://instats.gitblit.dev/hiitsme/"; + private final static Logger LOG = LoggerFactory.getLogger(GitblitInstance.class); + + private IRuntimeManager runtimeManager; + + private String instanceId; + + private GitblitInstanceReport report; + + private ScheduledExecutorService executor; + + + /** + * Initialize the Gitblit instance reporting system. + * + * This will gather the static and dynamic statistics about the running + * instance, so that they can be reported. + * + * @param runtimeManager + * The runtime manager is used to determine the type of instance + * as well as for some other settings and data. + */ + public void init(IRuntimeManager runtimeManager) { + this.runtimeManager = runtimeManager; + + // Initialize ID + GitblitInstanceId instanceId = new GitblitInstanceId(runtimeManager.getBaseFolder()); + this.instanceId = instanceId.getId().toString(); + LOG.info(this.instanceId); + + GitblitInstanceStat instanceStat; + + if (runtimeManager.getSettings().hasSettings("container.dockerfileVersion")) { + instanceStat = new GitblitInstanceStat(GitblitInstanceStat.GitblitInstanceType.DOCKER); + } + else if (runtimeManager.getStatus().isGO){ + instanceStat = new GitblitInstanceStat(GitblitInstanceStat.GitblitInstanceType.GO); + } + else { + instanceStat = new GitblitInstanceStat(GitblitInstanceStat.GitblitInstanceType.WAR); + } + + instanceStat.init(runtimeManager.getStatus()); + + this.report = new GitblitInstanceReport(this.instanceId, instanceStat); + } + + + public void start() + { + if (shouldRunReports()) { + startReports(); + } + } + + public void stop() + { + if (this.executor != null && !this.executor.isShutdown() && !this.executor.isTerminated()) { + this.executor.shutdownNow(); + System.out.println("Gitblit instance reporting task stopped."); + } + } + + + + /** + * Determine if the reporting task should run. + * + * We do not want to report anything, i.e. the reporting task to run, + * if we are running unit tests or integration tests. + * Instance reports should only be sent for production instances or released versions. + * Therefore we also check if the Gitblit version is a SNAPSHOT version, + * or if the docker image is not a release version, when running from a docker image. + * A docker image running under GOSS should also not report anything. + */ + boolean shouldRunReports() + { + // We can only run reports when we have been initialized + if (this.report == null || this.runtimeManager == null) { + return false; + } + + // Check if we are running in a test environment + IStoredSettings settings = this.runtimeManager.getSettings(); + if (! settings.getString("gitblit.testReportingUrl", "").isEmpty()) { + // Force reporting to run overriding any test settings + LOG.debug("Enabled reporting to test server URL: {}", settings.getString("gitblit.testReportingUrl", "")); + return true; + } + if (settings.getBoolean("gitblit.testRun", false)) { + return false; + } + + // Check if we are running a SNAPSHOT version + if (this.runtimeManager.getStatus().version.endsWith("SNAPSHOT")) { + return false; + } + + if (this.report.instanceStat.instanceType == GitblitInstanceStat.GitblitInstanceType.DOCKER) { + // Check if we are running a docker image that is not a release version + if (! settings.getString("container.imageType", "").equals("release")) { + return false; + } + + // Check if we are running a docker image under GOSS + if (System.getenv("GITBLIT_GOSS_TEST") != null) { + return false; + } + } + + return true; + } + + + /** + * Start the reporting task. + * + * This will start a thread that runs once a day and sends the instance + * report to the popularity report server. + */ + private void startReports() + { + this.executor = Executors.newSingleThreadScheduledExecutor(); + + String statsUrl = STATS_URL; + int delay = 24; + int period = 24 * 60; // 24 hours in minutes + TimeUnit unit = TimeUnit.MINUTES; + long retryInterval = 60 * 60 * 1000; // 1 hour in milliseconds + final long retryTimeout = 20 * 60 * 60 * 1000; // 20 hours in milliseconds + + // If we are running in a test environment, we will send the reports more frequently + String testUrl = this.runtimeManager.getSettings().getString("gitblit.testReportingUrl", ""); + if (! testUrl.isEmpty()) { + statsUrl = testUrl; + delay = 10; + period = 24; + unit = TimeUnit.SECONDS; + retryInterval = 10 * 1000; // 10 seconds in milliseconds + } + + final String baseUrl = statsUrl; + final long retryIntervalFinal = retryInterval; + this.executor.scheduleAtFixedRate(new Runnable() + { + @Override + public void run() + { + sendMyStats(baseUrl + instanceId, retryIntervalFinal, retryTimeout); + } + }, delay, period, unit); + } + + /** + * Send the instance report to the popularity report server. + * + * This will send a JSON object to the server with the instance report. + * + * @param reportUrl + * The URL to send the report to. + * @param retryInterval + * The interval in milliseconds to wait before retrying to send the report if it failed. + * @param retryTimeout + * The timeout in milliseconds to give up sending the report if it fails repeatedly. + */ + private void sendMyStats(String reportUrl, long retryInterval, long retryTimeout) + { + // Create a HTTP POST request payload + String report = JsonUtils.toJsonString(this.report.fromNow()); + + int status = 0; + long timeToGiveup = System.currentTimeMillis() + retryTimeout; + while (status != 200 && System.currentTimeMillis() < timeToGiveup) { + try { + status = sendJsonString(reportUrl, report, "gitblitta", "countmein".toCharArray()); + if (status != 200) { + LOG.debug("Error sending stats to " + reportUrl + ": " + status); + } + } + catch (IOException e) { + LOG.debug("Exception sending stats to " + reportUrl + ": " + e.getMessage()); + } + + if (status != 200) { + try { + Thread.sleep(retryInterval); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; // exit if interrupted + } + } + } + } + +} diff --git a/src/main/java/com/gitblit/instance/GitblitInstanceId.java b/src/main/java/com/gitblit/instance/GitblitInstanceId.java new file mode 100644 index 00000000..652c200c --- /dev/null +++ b/src/main/java/com/gitblit/instance/GitblitInstanceId.java @@ -0,0 +1,265 @@ +package com.gitblit.instance; + +import com.gitblit.utils.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.Socket; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.UUID; + +/** + * The instance id is a unique identifier for an installed Gitblit instance. + * + * This is used to track the number of Gitblit instances in the field. + * Its purpose is to gauge the popularity of Gitblit and to help + * prioritize feature requests. + * + * The instance id should be unique between different instances, even + * on the same machine. But it should stay the same between restarts of + * the same instance. It should also stay the same between upgrades of + * the same instance. Therefore, it must be stored in a file that is + * not overwritten during upgrades, once it has been created. + */ +public class GitblitInstanceId +{ + static final String STORAGE_FILE = "gbins"; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final File idFileBase; + + private UUID id; + + + /** + * Constructor. + */ + public GitblitInstanceId() + { + this.idFileBase = null; + } + + /** + * Constructor. + */ + public GitblitInstanceId(File idFileBase) + { + this.idFileBase = idFileBase; + } + + + /** + * Get the instance id. + * + * @return the instance id. + */ + public UUID getId() { + if (this.id == null) { + load(); + } + return this.id; + } + + + /** + * Load the instance id from the file. + */ + private void load() + { + if (this.idFileBase == null) { + // Not working with stored id. + log.debug("No id file base directory specified. Generated id is not persisted."); + generate(); + return; + } + + File idFile = new File(this.idFileBase, STORAGE_FILE); + if (idFile.exists()) { + // Read the file + String uuidString = readFromFile(idFile); + + // Parse the UUID + try { + this.id = UUID.fromString(uuidString); + return; + } + catch (IllegalArgumentException e) { + log.debug("Unable to parse instance id. Will generate a new one: {}", e.getMessage(), e); + } + } + + // Generate a new instance id and persist it to disk. + generate(); + storeToFile(idFile); + } + + + private String readFromFile(File idfile) + { +// log.debug("Loading instance id from file: {}", idfile.getAbsolutePath()); + + String string = FileUtils.readContent(idfile, null).trim(); + String uuidString = string.replaceAll("\\s+",""); + return uuidString.trim(); + } + + private void storeToFile(File idfile) + { + // Make sure that the directory exists + if (!idfile.getParentFile().exists()) { + if (!idfile.getParentFile().mkdirs()) { + log.debug("Unable to create directory for instance id file: {}", idfile.getParentFile().getAbsolutePath()); + return; + } + } + + // Write the UUID to the file + String uuidString = this.id.toString(); + FileUtils.writeContent(idfile, uuidString); + } + + + /** + * Generate a new instance id and persist it to disk. + * + * UUID is variant, i.e. OSF DCE, version 8, a custom format. + * xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + * date -rand-8rnd-8rnd- rand OUI + * + * The variant nibble has the variant (1) in the upper two bits as 0b10xx, + * and the lower two bits are used as a version, currently 0bxx00. + * Should the format of this UUID change, the version can be incremented + * to 0bxx01 or 0bxx10. Further increments would set the bits in the variant + * nibble to 0bxx11 and employ more bits from the next nibble for further + * differentiation. + */ + private void generate() + { + // Start with a random UUID + UUID id = UUID.randomUUID(); + long upper = id.getMostSignificantBits(); + long lower = id.getLeastSignificantBits(); + + + // Set the variant bits to 0b1000, variant 1, our version 0. + lower &= 0x0FFFFFFFFFFFFFFFL; // Clear the variant bits + lower |= 0x8000000000000000L; // Set the variant bits to 0b1000 + + // Set the version bits to 0b1000, version 8. + upper &= 0xFFFFFFFFFFFF0FFFL; // Clear the version bits + upper |= 0x0000000000008000L; // Set the version bits to 0b1000 + + + // Set the first four bytes to represent the date. + long date = System.currentTimeMillis(); + date &= 0xFFFFFFFFFFFF0000L; // Clear the last two bytes, those are only a few minutes. + date <<= 2 * 8; // We do not need the upper two bytes, that is too far into the future. + + upper &= 0x00000000FFFFFFFFL; // Clear the date bits. + upper |= date; // Set the date in the upper 32 bits. + + + // Set the OUI in the lower three bytes. + Long oui = getNodeOUI(); + if (oui != null) { + lower &= 0xFFFFFFFFFF000000L; // Clear the OUI bits. + lower |= (0x1000000L | oui); // Set the OUI in the lower three bytes. Mark as valid OUI in bit above them. + } + else { + // Mark this as an invalid OUI, i.e. random bits, by setting the bit above the OUI bits to zero. + lower &= 0xFFFFFFFFFEFFFFFFL; // Clear the valid OUI indicator bit. + } + + this.id = new UUID(upper, lower); + } + + + /** + * Get the OUI of one NIC of this host. + * + * @return null if no OUI could be detected, otherwise the OUI in the lower three bytes of a Long. + */ + private Long getNodeOUI() + { + byte[] node = null; + String logPrefix = "Unable to detect host. Use random value."; + + try { + InetAddress ipa = InetAddress.getLocalHost(); + NetworkInterface iface = NetworkInterface.getByInetAddress(ipa); + if (iface != null) { + node = iface.getHardwareAddress(); + logPrefix = "From getLocalHost:"; + } + + if (node == null) { + List<byte[]> macs = new ArrayList<>(); + List<byte[]> offmacs = new ArrayList<>(); + Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces(); + while (interfaces.hasMoreElements()) { + iface = interfaces.nextElement(); + byte[] mac = iface.getHardwareAddress(); + if (mac != null) { + if (iface.isLoopback()) { + continue; + } + if (iface.isVirtual() || iface.isPointToPoint()) { + continue; + } + if (iface.isUp()) { + macs.add(mac); + } + else { + offmacs.add(mac); + } + } + } + + if (macs.size() == 1) { + node = macs.get(0); + logPrefix = "From up iface:"; + } + else if (offmacs.size() == 1) { + node = offmacs.get(0); + logPrefix = "From down iface:"; + } + } + + if (node == null) { + Socket socket = new Socket("www.gitblit.dev", 80); + ipa = socket.getLocalAddress(); + socket.close(); + iface = NetworkInterface.getByInetAddress(ipa); + if (iface != null) { + node = iface.getHardwareAddress(); + logPrefix = "From socket:"; + } + } + + if (node == null) { + log.debug(logPrefix); + return null; + } + + if (log.isDebugEnabled()) { + log.debug("{} {}", logPrefix, String.format("%02X:%02X:%02X", node[0], node[1], node[2])); + } + + long l = (((long)node[0]) << 16) & 0xff0000; + l |= (((long)node[1]) << 8) & 0xff00; + l |= ((long)node[2]) & 0xff; + return l; + } + catch (IOException e) { + log.debug("Exception while getting OUI: {}", e.getMessage(), e); + return null; + } + } +} diff --git a/src/main/java/com/gitblit/instance/GitblitInstanceReport.java b/src/main/java/com/gitblit/instance/GitblitInstanceReport.java new file mode 100644 index 00000000..7b8db370 --- /dev/null +++ b/src/main/java/com/gitblit/instance/GitblitInstanceReport.java @@ -0,0 +1,43 @@ +package com.gitblit.instance; + +import com.google.gson.annotations.SerializedName; + +import java.text.SimpleDateFormat; +import java.util.TimeZone; + +/** + * GitblitInstanceReport collects the static and dynamic statistics about a running + * Gitblit instance, pairs it with a report version and instance id. + * This can then be send to the popularity report server. + * + */ +class GitblitInstanceReport +{ + private final int reportVersion = 1; + @SerializedName("instance") + private final String instanceId; + private final String startTs; + private String lpingTs; + + final GitblitInstanceStat instanceStat; + + GitblitInstanceReport(String instanceId, GitblitInstanceStat instanceStat) + { + this.instanceId = instanceId; + this.instanceStat = instanceStat; + + // Convert the timestamp taken from instanceStat to a string in the format "yyyy-MM-dd'T'HHmmssZ" so + // it can be used better in a file name. It is replicated here so that it can be directly used by the receiver. + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + this.startTs = dateFormat.format(instanceStat.startTs); + } + + GitblitInstanceReport fromNow() + { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + this.lpingTs = dateFormat.format(System.currentTimeMillis()); + return this; + } +} diff --git a/src/main/java/com/gitblit/instance/GitblitInstanceStat.java b/src/main/java/com/gitblit/instance/GitblitInstanceStat.java new file mode 100644 index 00000000..15ae6fa7 --- /dev/null +++ b/src/main/java/com/gitblit/instance/GitblitInstanceStat.java @@ -0,0 +1,127 @@ +package com.gitblit.instance; + +import com.gitblit.models.ServerStatus; + +import java.util.Date; + +/** + * GitblitInstanceStat collects the static information about a Gitblit instance, + * such as its version, type, operating system and other static data. + * + */ +class GitblitInstanceStat +{ + + enum GitblitInstanceType { + GO, + WAR, + EXPRESS, + DOCKER + } + + final GitblitInstanceType instanceType; + + String version; + Date startTs; + String os; + String osName; + String osVersion; + String osArch; + String javaVersion; + String javaVendor; + String javaRuntimeVersion; + String javaRuntimeName; + String javaVmVersion; + String javaVmName; + long maxMem; + + + GitblitInstanceStat() + { + this.instanceType = GitblitInstanceType.WAR; + initOS(); + initJava(); + } + + GitblitInstanceStat(GitblitInstanceType instanceType) + { + this.instanceType = instanceType; + initOS(); + initJava(); + } + + + GitblitInstanceStat init(ServerStatus serverStatus) + { + this.version = serverStatus.version; + this.startTs = serverStatus.bootDate; + + this.maxMem = serverStatus.heapMaximum; + + return this; + } + + + void initOS() + { + String os = System.getProperty("os.name"); + if (os == null) { + this.os = "Unknown"; + } else { + String oslc = os.toLowerCase(); + if (oslc.contains("windows")) { + this.os = "Windows"; + } else if (oslc.contains("linux")) { + this.os = "Linux"; + } else if (oslc.contains("mac") || oslc.contains("darwin")) { + this.os = "macOS"; + } else if (oslc.contains("bsd")) { + this.os = "BSD"; + } else if (oslc.contains("solaris") || oslc.contains("sun") || + oslc.contains("aix") || oslc.contains("hpux") || oslc.contains("unix")) { + this.os = "Unix"; + } else { + this.os = os; + } + } + + this.osName = System.getProperty("os.name"); + this.osVersion = System.getProperty("os.version"); + this.osArch = System.getProperty("os.arch"); + } + + void initJava() + { + this.javaVersion = System.getProperty("java.version"); + this.javaVendor = System.getProperty("java.vendor"); + this.javaRuntimeVersion = System.getProperty("java.runtime.version", ""); + this.javaRuntimeName = System.getProperty("java.runtime.name", ""); + this.javaVmVersion = System.getProperty("java.vm.version", ""); + this.javaVmName = System.getProperty("java.vm.name", ""); + } + + + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append("GitblitInstanceStat {") + .append("\n instanceType: ").append(instanceType) + .append(",\n version: ").append(version) + .append(",\n startTs: ").append(startTs) + .append(",\n os: ").append(os) + .append(",\n osName: ").append(osName) + .append(",\n osVersion: ").append(osVersion) + .append(",\n osArch: ").append(osArch) + .append(",\n javaVersion: ").append(javaVersion) + .append(",\n javaVendor: ").append(javaVendor) + .append(",\n javaRuntimeVersion: ").append(javaRuntimeVersion) + .append(",\n javaRuntimeName: ").append(javaRuntimeName) + .append(",\n javaVmVersion: ").append(javaVmVersion) + .append(",\n javaVmName: ").append(javaVmName) + .append(",\n maxMem: ").append(maxMem) + .append("\n}"); + return sb.toString(); + } +} |